diff --git a/datavines-common/src/main/java/io/datavines/common/utils/DateUtils.java b/datavines-common/src/main/java/io/datavines/common/utils/DateUtils.java index d8f37b49a..37db7f2c9 100644 --- a/datavines-common/src/main/java/io/datavines/common/utils/DateUtils.java +++ b/datavines-common/src/main/java/io/datavines/common/utils/DateUtils.java @@ -43,6 +43,7 @@ public class DateUtils { public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss"; public static final String YYYYMMDD = "yyyyMMdd"; + public static final String YYYY_MM_DD = "yyyy-MM-dd"; /** * date to local datetime diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java index 4c5378deb..26df2133a 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java @@ -67,7 +67,7 @@ protected InputParam getPropertiesInput(boolean isEn) { return getInputParam("properties", isEn ? "properties" : "参数", isEn ? "please enter properties,like key=value&key1=value1" : "请填入参数，格式为key=value&key1=value1", 2, null, - "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true"); + "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true&allowPublicKeyRetrieval=true"); } @Override diff --git a/datavines-core/src/main/java/io/datavines/core/constant/DataVinesConstants.java b/datavines-core/src/main/java/io/datavines/core/constant/DataVinesConstants.java index 9eecf1a9a..8628dc569 100644 --- a/datavines-core/src/main/java/io/datavines/core/constant/DataVinesConstants.java +++ b/datavines-core/src/main/java/io/datavines/core/constant/DataVinesConstants.java @@ -83,4 +83,6 @@ public class DataVinesConstants { public static final String SPARK = "spark"; public static final String LOCALE_LANGUAGE_COOKIE = "language"; + + public static final String COMMON_TASK_TYPE = "commonTaskType"; } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java index 5500d01ad..2db8656d6 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java @@ -20,15 +20,15 @@ public class LoggerFactory { - private static Logger logger; + private static ThreadLocal logger = new ThreadLocal<>(); public static void setLogger(Logger newLogger){ - logger = newLogger; + logger.set(newLogger); } public static Logger getLogger(Class clazz) { - if (logger != null) { - return logger; + if (logger.get() != null) { + return logger.get(); } else { return org.slf4j.LoggerFactory.getLogger(clazz); } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java index 7d2af0a7a..14f6c5615 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java @@ -69,7 +69,6 @@ protected List getSourceConfigs() throws DataVinesException { connectorParameterMap.putAll(metricInputParameter); connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - metricInputParameter.put(DATABASE_NAME, metricInputParameter.get(DATABASE)); metricInputParameter.put(TABLE_NAME, metricInputParameter.get(TABLE)); metricInputParameter.put(COLUMN_NAME, metricInputParameter.get(COLUMN)); @@ -140,7 +139,6 @@ protected List getSourceConfigs() throws DataVinesException { connectorParameterMap.put(TABLE, table); connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - String outputTable = metricInputParameter.get(TABLE2); connectorParameterMap.put(OUTPUT_TABLE, outputTable); connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricDirectionType.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricDirectionType.java new file mode 100644 index 000000000..8a247bbc7 --- /dev/null +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricDirectionType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.metric.api; + +import com.fasterxml.jackson.annotation.JsonValue; +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; + +public enum MetricDirectionType { + /** + * 0-positive + * 1-negative + */ + POSITIVE(0,"positive"), + NEGATIVE(1,"negative"); + + MetricDirectionType(int code, String description) { + this.code = code; + this.description = description; + } + + @Getter + private final int code; + private final String description; + + @JsonValue + public String getDescription() { + return description; + } + + private static final Map VALUES_MAP = new HashMap<>(); + + static { + for (MetricDirectionType type : MetricDirectionType.values()) { + VALUES_MAP.put(type.code,type); + } + } + + public static MetricDirectionType of(Integer code) { + if (VALUES_MAP.containsKey(code)) { + return VALUES_MAP.get(code); + } + throw new IllegalArgumentException("invalid code : " + code); + } + +} \ No newline at end of file diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java index 7f999365a..bbe6a6d31 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java @@ -22,6 +22,7 @@ import org.apache.commons.collections4.MapUtils; import java.io.Serializable; +import java.math.BigDecimal; import java.util.Map; @Data @@ -29,15 +30,15 @@ public class MetricExecutionResult implements Serializable { private static final long serialVersionUID = -1L; - private Double actualValue; + private BigDecimal actualValue; - private Double expectedValue; + private BigDecimal expectedValue; private String resultFormula; private String operator; - private Double threshold; + private BigDecimal threshold; private String expectedType; @@ -62,11 +63,11 @@ public MetricExecutionResult(Map dataMap) { } if (dataMap.get("actual_value") != null) { - actualValue = Double.valueOf(String.valueOf(dataMap.get("actual_value")).trim()); + actualValue = new BigDecimal(String.valueOf(dataMap.get("actual_value")).trim()); } if (dataMap.get("expected_value") != null) { - expectedValue = Double.valueOf(String.valueOf(dataMap.get("expected_value")).trim()); + expectedValue = new BigDecimal(String.valueOf(dataMap.get("expected_value")).trim()); } if (dataMap.get("result_formula") != null) { @@ -78,7 +79,7 @@ public MetricExecutionResult(Map dataMap) { } if (dataMap.get("threshold") != null) { - threshold = Double.valueOf(String.valueOf(dataMap.get("threshold")).trim()); + threshold = new BigDecimal(String.valueOf(dataMap.get("threshold")).trim()); } if (dataMap.get("metric_name") != null) { diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java index 7e833cd3e..0acd5008a 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java @@ -32,8 +32,8 @@ public class MetricValidator { */ public static boolean isSuccess(MetricExecutionResult executionResult) { - Double actualValue = executionResult.getActualValue(); - Double expectedValue = executionResult.getExpectedValue(); + BigDecimal actualValue = executionResult.getActualValue(); + BigDecimal expectedValue = executionResult.getExpectedValue(); OperatorType operatorType = OperatorType.of(StringUtils.trim(executionResult.getOperator())); ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class) @@ -44,26 +44,36 @@ public static boolean isSuccess(MetricExecutionResult executionResult) { executionResult.getThreshold()); } - private static boolean getCompareResult(OperatorType operatorType, Double srcValue, Double targetValue) { + public static BigDecimal getQualityScore(MetricExecutionResult executionResult, boolean isSuccess) { + BigDecimal actualValue = executionResult.getActualValue(); + BigDecimal expectedValue = executionResult.getExpectedValue(); + + ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class) + .getOrCreatePlugin(executionResult.getResultFormula()); + SqlMetric metric = PluginLoader.getPluginLoader(SqlMetric.class) + .getOrCreatePlugin(executionResult.getMetricName()); + + return resultFormula.getScore(actualValue, expectedValue, isSuccess, metric.getDirectionType()); + } + + private static boolean getCompareResult(OperatorType operatorType, BigDecimal srcValue, BigDecimal targetValue) { if (srcValue == null || targetValue == null) { return false; } - BigDecimal src = BigDecimal.valueOf(srcValue); - BigDecimal target = BigDecimal.valueOf(targetValue); switch (operatorType) { case EQ: - return src.compareTo(target) == 0; + return srcValue.compareTo(targetValue) == 0; case LT: - return src.compareTo(target) <= -1; + return srcValue.compareTo(targetValue) <= -1; case LTE: - return src.compareTo(target) == 0 || src.compareTo(target) <= -1; + return srcValue.compareTo(targetValue) == 0 || srcValue.compareTo(targetValue) <= -1; case GT: - return src.compareTo(target) >= 1; + return srcValue.compareTo(targetValue) >= 1; case GTE: - return src.compareTo(target) == 0 || src.compareTo(target) >= 1; + return srcValue.compareTo(targetValue) == 0 || srcValue.compareTo(targetValue) >= 1; case NE: - return src.compareTo(target) != 0; + return srcValue.compareTo(targetValue) != 0; default: return true; } diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java index b96544999..9faa566cf 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java @@ -18,6 +18,8 @@ import io.datavines.spi.SPI; +import java.math.BigDecimal; + @SPI public interface ResultFormula { @@ -29,7 +31,9 @@ default String getNameByLanguage(boolean isEn) { return isEn ? getName() : getZhName(); } - Double getResult(Double actualValue, Double expectedValue); + BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue); + + BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction); String getResultFormat(boolean isEn); diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java index 5434c9940..46e847459 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java @@ -92,4 +92,8 @@ default boolean supportMultiple() { default List> getMetricParameter(Map metricParameter) { return Collections.singletonList(metricParameter); } + + default MetricDirectionType getDirectionType() { + return MetricDirectionType.POSITIVE; + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java old mode 100755 new mode 100644 diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java index 826fb3edf..dd0e2b50f 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java @@ -48,7 +48,7 @@ public String getKey(Map inputParameter) { public String getExecuteSql(Map inputParameter) { String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); String newKey = "expected_value_" + uniqueKey; - return sql.toString().replace("expected_value",newKey); + return sql.toString().replace("expected_value", newKey); } @Override diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java old mode 100755 new mode 100644 diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java index 0cb271462..d5ef981d0 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java @@ -18,10 +18,11 @@ import io.datavines.common.enums.DataVinesDataType; import io.datavines.metric.api.MetricDimension; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -66,6 +67,11 @@ public void prepare(Map config) { @Override public List suitableType() { - return Arrays.asList(DataVinesDataType.STRING_TYPE); + return Collections.singletonList(DataVinesDataType.STRING_TYPE); + } + + @Override + public MetricDirectionType getDirectionType() { + return MetricDirectionType.NEGATIVE; } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java index 7e1c6425a..a485db28f 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java @@ -18,6 +18,7 @@ import io.datavines.common.enums.DataVinesDataType; import io.datavines.metric.api.MetricDimension; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; import org.apache.commons.lang3.StringUtils; @@ -75,4 +76,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public MetricDirectionType getDirectionType() { + return MetricDirectionType.NEGATIVE; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java index bca94f17d..611dd3806 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java @@ -19,6 +19,7 @@ import io.datavines.common.enums.DataVinesDataType; import io.datavines.metric.api.ConfigItem; import io.datavines.metric.api.MetricDimension; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; @@ -74,4 +75,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public MetricDirectionType getDirectionType() { + return MetricDirectionType.NEGATIVE; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java index 7e5f2502a..3aeee04e4 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java @@ -68,6 +68,7 @@ public void prepare(Map config) { } super.prepare(config); } + @Override public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java index e5a7890c7..dab9c6b49 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java @@ -19,6 +19,7 @@ import io.datavines.common.enums.DataVinesDataType; import io.datavines.metric.api.ConfigItem; import io.datavines.metric.api.MetricDimension; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; @@ -74,4 +75,9 @@ public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + @Override + public MetricDirectionType getDirectionType() { + return MetricDirectionType.NEGATIVE; + } + } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java index ce349116a..5d6a58950 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java @@ -59,7 +59,6 @@ public boolean isInvalidateItemsCanOutput() { @Override public void prepare(Map config) { - if (config.containsKey("column")) { filters.add(" ${column} is not null "); } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java index 2bc3d5fb2..cffb1c595 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java @@ -18,6 +18,7 @@ import io.datavines.common.enums.DataVinesDataType; import io.datavines.metric.api.MetricDimension; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; @@ -69,4 +70,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public MetricDirectionType getDirectionType() { + return MetricDirectionType.NEGATIVE; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java old mode 100755 new mode 100644 diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java index d3a9a94c5..0a3f35b1a 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java @@ -16,9 +16,12 @@ */ package io.datavines.metric.result.formula; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.ResultFormula; import io.datavines.metric.api.ResultFormulaType; +import java.math.BigDecimal; + public class Count implements ResultFormula { @Override @@ -32,10 +35,15 @@ public String getZhName() { } @Override - public Double getResult(Double actualValue, Double expectedValue) { + public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) { return actualValue; } + @Override + public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction) { + return isSuccess ? new BigDecimal(100) : new BigDecimal(0) ; + } + @Override public String getResultFormat(boolean isEn) { return isEn ? "Actual(${actual_value})" : "实际值(${actual_value})"; diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java index 8aa0430f0..68fdd73c6 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java @@ -16,6 +16,7 @@ */ package io.datavines.metric.result.formula; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.ResultFormula; import io.datavines.metric.api.ResultFormulaType; @@ -34,10 +35,13 @@ public String getZhName() { } @Override - public Double getResult(Double actualValue, Double expectedValue) { - BigDecimal av = new BigDecimal(Double.toString(actualValue)); - BigDecimal bv = new BigDecimal(Double.toString(expectedValue)); - return av.subtract(bv).doubleValue(); + public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) { + return actualValue.subtract(expectedValue); + } + + @Override + public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction) { + return isSuccess ? new BigDecimal(100) : new BigDecimal(0) ; } @Override diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java index cf3f1f6d1..ca87996d9 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java @@ -16,9 +16,13 @@ */ package io.datavines.metric.result.formula; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.ResultFormula; import io.datavines.metric.api.ResultFormulaType; +import java.math.BigDecimal; +import java.math.RoundingMode; + public class DiffPercentage implements ResultFormula { @Override @@ -32,15 +36,21 @@ public String getZhName() { } @Override - public Double getResult(Double actualValue, Double expectedValue) { - double result = 0; - if (expectedValue > 0) { - result = Math.abs(expectedValue - actualValue) / expectedValue * 100; + public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) { + BigDecimal result = BigDecimal.valueOf(0); + if (expectedValue != null) { + BigDecimal quotient = actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP); + result = quotient.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); } return result; } + @Override + public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction) { + return getResult(actualValue, expectedValue); + } + @Override public String getResultFormat(boolean isEn) { return isEn? "|Actual(${actual_value})-Expected(${expected_value})|/Expected(${expected_value}) x 100%" : diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java index 71299a9d0..51e2fc986 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java @@ -16,9 +16,12 @@ */ package io.datavines.metric.result.formula; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.ResultFormula; import io.datavines.metric.api.ResultFormulaType; +import java.math.BigDecimal; + public class Diff implements ResultFormula { @Override @@ -32,8 +35,13 @@ public String getZhName() { } @Override - public Double getResult(Double actualValue, Double expectedValue) { - return Math.abs(actualValue - expectedValue); + public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) { + return actualValue.subtract(expectedValue).abs(); + } + + @Override + public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction) { + return isSuccess ? new BigDecimal(100) : new BigDecimal(0) ; } @Override diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java index bb272ba2f..561407f14 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java @@ -16,9 +16,13 @@ */ package io.datavines.metric.result.formula; +import io.datavines.metric.api.MetricDirectionType; import io.datavines.metric.api.ResultFormula; import io.datavines.metric.api.ResultFormulaType; +import java.math.BigDecimal; +import java.math.RoundingMode; + public class Percentage implements ResultFormula { @Override @@ -32,10 +36,24 @@ public String getZhName() { } @Override - public Double getResult(Double actualValue, Double expectedValue) { - double result = 0; - if (expectedValue > 0) { - result = actualValue / expectedValue * 100; + public BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue) { + BigDecimal result = BigDecimal.valueOf(0); + if (expectedValue != null) { + result = actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); + } + + return result; + } + + @Override + public BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction) { + BigDecimal result = BigDecimal.valueOf(0); + if (expectedValue != null) { + if (MetricDirectionType.NEGATIVE == direction) { + result = new BigDecimal(100).subtract(actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100))); + } else if (MetricDirectionType.POSITIVE == direction) { + result = actualValue.divide(expectedValue, 2, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); + } } return result; diff --git a/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java b/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java index ced622e06..c5cd79f48 100644 --- a/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java +++ b/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java @@ -23,9 +23,9 @@ import io.datavines.common.utils.ThreadUtils; import io.datavines.core.constant.DataVinesConstants; import io.datavines.registry.api.Registry; -import io.datavines.server.catalog.metadata.CatalogMetaDataFetchTaskManager; -import io.datavines.server.catalog.metadata.CatalogMetaDataFetchTaskScheduler; -import io.datavines.server.catalog.metadata.CatalogMetaDataFetchTaskFailover; +import io.datavines.server.scheduler.CommonTaskManager; +import io.datavines.server.scheduler.CommonTaskScheduler; +import io.datavines.server.scheduler.CommonTaskFailover; import io.datavines.server.dqc.coordinator.cache.JobExecutionResponseProcessor; import io.datavines.server.registry.Register; import io.datavines.server.dqc.coordinator.cache.JobExecuteManager; @@ -82,10 +82,10 @@ private void initializeAndStart() throws Exception { JobExecutionResponseProcessor.getInstance().setJobExecuteManager(jobExecuteManager); - CatalogMetaDataFetchTaskManager catalogMetaDataFetchTaskManager = new CatalogMetaDataFetchTaskManager(); - catalogMetaDataFetchTaskManager.start(); + CommonTaskManager commonTaskManager = new CommonTaskManager(); + commonTaskManager.start(); - CatalogMetaDataFetchTaskFailover catalogMetaDataFetchTaskFailover = new CatalogMetaDataFetchTaskFailover(catalogMetaDataFetchTaskManager); + CommonTaskFailover commonTaskFailover = new CommonTaskFailover(commonTaskManager); jobExecutionFailover = new JobExecutionFailover(jobExecuteManager); @@ -96,15 +96,15 @@ private void initializeAndStart() throws Exception { registry.init(CommonPropertyUtils.getProperties()); registryHolder.setRegistry(registry); - register = new Register(registry, jobExecutionFailover, catalogMetaDataFetchTaskFailover); + register = new Register(registry, jobExecutionFailover, commonTaskFailover); register.start(); //start job scheduler JobScheduler jobScheduler = new JobScheduler(jobExecuteManager, register); jobScheduler.start(); - CatalogMetaDataFetchTaskScheduler catalogMetaDataFetchTaskScheduler = new CatalogMetaDataFetchTaskScheduler(catalogMetaDataFetchTaskManager, register); - catalogMetaDataFetchTaskScheduler.start(); + CommonTaskScheduler commonTaskScheduler = new CommonTaskScheduler(commonTaskManager, register); + commonTaskScheduler.start(); Runtime.getRuntime().addShutdownHook(new Thread(() -> close("shutdownHook"))); } diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java index 2276bf615..6688051da 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java @@ -49,7 +49,7 @@ public class CatalogController { @Autowired - private CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService; + private CommonTaskService commonTaskService; @Autowired private CatalogEntityInstanceService catalogEntityInstanceService; @@ -69,7 +69,7 @@ public class CatalogController { @ApiOperation(value = "refresh", response = Long.class) @PostMapping(value = "/refresh", consumes = MediaType.APPLICATION_JSON_VALUE) public Object refreshCatalog(@Valid @RequestBody CatalogRefresh catalogRefresh) { - return catalogMetaDataFetchTaskService.refreshCatalog(catalogRefresh); + return commonTaskService.refreshCatalog(catalogRefresh); } @ApiOperation(value = "execute data profile job", response = Long.class) diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskController.java similarity index 72% rename from datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataController.java rename to datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskController.java index 416974238..9d67f0de9 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskController.java @@ -18,7 +18,7 @@ import io.datavines.core.aop.RefreshToken; import io.datavines.core.constant.DataVinesConstants; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskService; +import io.datavines.server.repository.service.CommonTaskService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; @@ -26,21 +26,22 @@ import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; -@Api(value = "catalog", tags = "catalog", produces = MediaType.APPLICATION_JSON_VALUE) +@Api(value = "common-task", tags = "common-task", produces = MediaType.APPLICATION_JSON_VALUE) @RestController -@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/catalog/metadata/task", produces = MediaType.APPLICATION_JSON_VALUE) +@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/common-task", produces = MediaType.APPLICATION_JSON_VALUE) @RefreshToken @Validated -public class CatalogMetaDataController { +public class CommonTaskController { @Autowired - private CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService; + private CommonTaskService commonTaskService; - @ApiOperation(value = "get job page") + @ApiOperation(value = "get task page") @GetMapping(value = "/page") public Object page(@RequestParam("datasourceId") Long datasourceId, + @RequestParam("taskType") String taskType, @RequestParam("pageNumber") Integer pageNumber, @RequestParam("pageSize") Integer pageSize) { - return catalogMetaDataFetchTaskService.getFetchTaskPage(datasourceId, pageNumber, pageSize); + return commonTaskService.getFetchTaskPage(datasourceId, taskType.toLowerCase(), pageNumber, pageSize); } } \ No newline at end of file diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataScheduleController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskScheduleController.java similarity index 61% rename from datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataScheduleController.java rename to datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskScheduleController.java index 2c28c5990..57edf0b86 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogMetaDataScheduleController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/CommonTaskScheduleController.java @@ -19,9 +19,9 @@ import io.datavines.core.aop.RefreshToken; import io.datavines.core.constant.DataVinesConstants; import io.datavines.core.exception.DataVinesServerException; -import io.datavines.server.api.dto.bo.catalog.metadata.CatalogMetaDataFetchTaskScheduleCreateOrUpdate; +import io.datavines.server.api.dto.bo.task.CommonTaskScheduleCreateOrUpdate; import io.datavines.server.api.dto.bo.job.schedule.MapParam; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskScheduleService; +import io.datavines.server.repository.service.CommonTaskScheduleService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; @@ -33,29 +33,29 @@ @Api(value = "catalog", tags = "catalog", produces = MediaType.APPLICATION_JSON_VALUE) @RestController -@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/catalog/metadata/schedule", produces = MediaType.APPLICATION_JSON_VALUE) +@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/common-task/schedule", produces = MediaType.APPLICATION_JSON_VALUE) @RefreshToken @Validated -public class CatalogMetaDataScheduleController { +public class CommonTaskScheduleController { @Autowired - private CatalogMetaDataFetchTaskScheduleService catalogMetaDataFetchTaskScheduleService; + private CommonTaskScheduleService commonTaskScheduleService; - @ApiOperation(value = "create or update catalog metadata fetch task schedule") + @ApiOperation(value = "create or update common task schedule") @PostMapping(value = "/createOrUpdate",consumes = MediaType.APPLICATION_JSON_VALUE) - public Object createOrUpdateJob(@Valid @RequestBody CatalogMetaDataFetchTaskScheduleCreateOrUpdate taskScheduleCreateOrUpdate) throws DataVinesServerException { - return catalogMetaDataFetchTaskScheduleService.createOrUpdate(taskScheduleCreateOrUpdate); + public Object createOrUpdateJob(@Valid @RequestBody CommonTaskScheduleCreateOrUpdate taskScheduleCreateOrUpdate) throws DataVinesServerException { + return commonTaskScheduleService.createOrUpdate(taskScheduleCreateOrUpdate); } - @ApiOperation(value = "get metadata fetch task schedule by datasource id") - @GetMapping(value = "/{dataSourceId}") - public Object getByDataSourceId(@PathVariable Long dataSourceId) { - return catalogMetaDataFetchTaskScheduleService.getByDataSourceId(dataSourceId); + @ApiOperation(value = "get common task schedule by datasource id") + @GetMapping(value = "/{dataSourceId}/{taskType}") + public Object getByDataSourceId(@PathVariable String taskType, @PathVariable Long dataSourceId) { + return commonTaskScheduleService.getByDataSourceId(dataSourceId, taskType); } @ApiOperation(value = "get crontab expression by cycle") @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE, value = "/cron") public Object showCron(@Valid @RequestBody MapParam mapParam) throws DataVinesServerException { - return catalogMetaDataFetchTaskScheduleService.getCron(mapParam); + return commonTaskScheduleService.getCron(mapParam); } } \ No newline at end of file diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/JobExecutionController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/JobExecutionController.java old mode 100755 new mode 100644 diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/JobQualityReportController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/JobQualityReportController.java new file mode 100644 index 000000000..24bff4b23 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/JobQualityReportController.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.api.controller; + +import io.datavines.common.entity.job.SubmitJob; +import io.datavines.core.aop.RefreshToken; +import io.datavines.core.constant.DataVinesConstants; +import io.datavines.core.exception.DataVinesServerException; +import io.datavines.server.api.dto.bo.job.JobExecutionDashboardParam; +import io.datavines.server.api.dto.bo.job.JobExecutionPageParam; +import io.datavines.server.api.dto.bo.job.JobQualityReportDashboardParam; +import io.datavines.server.api.dto.vo.JobExecutionResultVO; +import io.datavines.server.repository.entity.JobExecution; +import io.datavines.server.repository.service.JobExecutionErrorDataService; +import io.datavines.server.repository.service.JobExecutionResultService; +import io.datavines.server.repository.service.JobExecutionService; +import io.datavines.server.repository.service.JobQualityReportService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.*; + +import javax.validation.Valid; + +@Slf4j +@Api(value = "job-quality-report", tags = "job-quality-report", produces = MediaType.APPLICATION_JSON_VALUE) +@RestController +@RequestMapping(value = DataVinesConstants.BASE_API_PATH + "/job/quality-report", produces = MediaType.APPLICATION_JSON_VALUE) +@RefreshToken +public class JobQualityReportController { + + @Autowired + private JobExecutionService jobExecutionService; + + @Autowired + private JobExecutionResultService jobExecutionResultService; + + @Autowired + private JobExecutionErrorDataService jobExecutionErrorDataService; + + @Autowired + private JobQualityReportService jobQualityReportService; + + @ApiOperation(value = "get job quality report page", response = JobExecutionResultVO.class, responseContainer = "page") + @PostMapping(value = "/page") + public Object page(@Valid @RequestBody JobQualityReportDashboardParam dashboardParam) { + return jobQualityReportService.getQualityReportPage(dashboardParam); + } + + @ApiOperation(value = "get job quality report page", response = JobExecutionResultVO.class, responseContainer = "list") + @GetMapping(value = "/listColumnExecution") + public Object listColumnExecution(@RequestParam Long reportId) { + return jobQualityReportService.listColumnExecution(reportId); + } + + @ApiOperation(value = "get job quality report score", response = JobExecutionResultVO.class) + @PostMapping(value = "/score") + public Object getScoreByCondition(@Valid @RequestBody JobQualityReportDashboardParam dashboardParam) { + return jobQualityReportService.getScoreByCondition(dashboardParam); + } + + @ApiOperation(value = "get job quality report score trend", response = JobExecutionResultVO.class) + @PostMapping(value = "/score-trend") + public Object getScoreTrendByCondition(@Valid @RequestBody JobQualityReportDashboardParam dashboardParam) { + return jobQualityReportService.getScoreTrendByCondition(dashboardParam); + } +} diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/CatalogRefresh.java b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/CatalogRefresh.java index b4cb88a67..a7ee454c8 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/CatalogRefresh.java +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/CatalogRefresh.java @@ -16,6 +16,7 @@ */ package io.datavines.server.api.dto.bo.catalog; +import io.datavines.server.enums.CommonTaskType; import lombok.Data; import javax.validation.constraints.NotNull; @@ -27,6 +28,8 @@ public class CatalogRefresh { @NotNull(message = "DataSourceId cannot be empty") private Long datasourceId; + private CommonTaskType taskType; + private String database; private String table; diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobExecutionDashboardParam.java b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobExecutionDashboardParam.java old mode 100755 new mode 100644 diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobExecutionPageParam.java b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobExecutionPageParam.java old mode 100755 new mode 100644 diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobQualityReportDashboardParam.java b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobQualityReportDashboardParam.java new file mode 100644 index 000000000..f5616205e --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/job/JobQualityReportDashboardParam.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.api.dto.bo.job; + +import lombok.Data; + +import javax.validation.constraints.NotNull; + +@Data +@NotNull(message = "JobQualityReportDashboardParam cannot be null") +public class JobQualityReportDashboardParam { + + @NotNull(message = "datasourceId can not be null") + private Long datasourceId; + + private String schemaName; + + private String tableName; + + private String startTime; + + private String endTime; + + private String reportDate; + + private int pageNumber; + + private int pageSize; +} diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/metadata/CatalogMetaDataFetchTaskScheduleCreateOrUpdate.java b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/task/CommonTaskScheduleCreateOrUpdate.java similarity index 83% rename from datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/metadata/CatalogMetaDataFetchTaskScheduleCreateOrUpdate.java rename to datavines-server/src/main/java/io/datavines/server/api/dto/bo/task/CommonTaskScheduleCreateOrUpdate.java index b89091220..20b6985db 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/dto/bo/catalog/metadata/CatalogMetaDataFetchTaskScheduleCreateOrUpdate.java +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/bo/task/CommonTaskScheduleCreateOrUpdate.java @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.api.dto.bo.catalog.metadata; +package io.datavines.server.api.dto.bo.task; import com.fasterxml.jackson.annotation.JsonFormat; import io.datavines.server.api.dto.bo.job.schedule.MapParam; +import io.datavines.server.enums.CommonTaskType; import lombok.Data; import javax.validation.constraints.NotBlank; @@ -25,11 +26,14 @@ import java.time.LocalDateTime; @Data -@NotNull(message = "Catalog MetaData Fetch Task Schedule Create Or Update cannot be null") -public class CatalogMetaDataFetchTaskScheduleCreateOrUpdate { +@NotNull(message = "Common Task Schedule Create Or Update cannot be null") +public class CommonTaskScheduleCreateOrUpdate { private Long id; + @NotNull(message = "task type cannot be empty") + private CommonTaskType taskType; + @NotNull(message = "DataSource id cannot be empty") private Long dataSourceId; diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionResultVO.java b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionResultVO.java index a9595247b..d2a83a41b 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionResultVO.java +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobExecutionResultVO.java @@ -19,6 +19,8 @@ import lombok.Data; import java.io.Serializable; +import java.math.BigDecimal; +import java.time.LocalDateTime; import java.util.Map; @Data @@ -36,4 +38,7 @@ public class JobExecutionResultVO implements Serializable { private String resultFormulaFormat; + private BigDecimal score; + + private LocalDateTime executionTime; } diff --git a/datavines-server/src/main/java/io/datavines/server/enums/StorageType.java b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScore.java similarity index 80% rename from datavines-server/src/main/java/io/datavines/server/enums/StorageType.java rename to datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScore.java index 63c849345..5d1500a40 100644 --- a/datavines-server/src/main/java/io/datavines/server/enums/StorageType.java +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScore.java @@ -14,18 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.enums; +package io.datavines.server.api.dto.vo; -/** - * data base types - */ -public enum StorageType { - /** - * 0 hdfs - * 1 s3 - * 2 none - */ - HDFS, - S3, - NONE +import lombok.Data; + +import java.math.BigDecimal; + +@Data +public class JobQualityReportScore { + + private BigDecimal score; + + private String qualityLevel; } diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScoreTrend.java b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScoreTrend.java new file mode 100644 index 000000000..8dcde8eb0 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportScoreTrend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.api.dto.vo; + +import lombok.Data; + +import java.math.BigDecimal; +import java.util.List; + +@Data +public class JobQualityReportScoreTrend { + + private List dateList; + + private List scoreList; +} diff --git a/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportVO.java b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportVO.java new file mode 100644 index 000000000..7fc0ef2aa --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/api/dto/vo/JobQualityReportVO.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.api.dto.vo; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.datavines.common.enums.ExecutionStatus; +import io.datavines.common.enums.JobType; +import io.datavines.core.utils.LanguageUtils; +import io.datavines.server.enums.DqJobExecutionState; +import lombok.Data; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; + +@Data +public class JobQualityReportVO implements Serializable { + + private static final long serialVersionUID = -1L; + + private Long id; + + private Long datasourceId; + + private String databaseName; + + private String tableName; + + private String columnName; + + private BigDecimal score; + + @JsonFormat(pattern="yyyy-MM-dd",timezone = "GMT+8") + private LocalDate reportDate; +} diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/cache/JobExecuteManager.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/cache/JobExecuteManager.java index 41320ba7e..384f97f40 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/cache/JobExecuteManager.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/cache/JobExecuteManager.java @@ -37,7 +37,7 @@ import io.datavines.server.dqc.command.JobExecuteAckCommand; import io.datavines.server.dqc.command.JobExecuteResponseCommand; import io.datavines.common.exception.DataVinesException; -import io.datavines.server.dqc.coordinator.operator.JobResultValidator; +import io.datavines.server.dqc.coordinator.validator.JobResultValidator; import io.datavines.server.repository.entity.DataSource; import io.datavines.server.repository.entity.Job; import io.datavines.server.repository.entity.JobExecution; diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/CatalogTaskScheduleJob.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/CatalogTaskScheduleJob.java index d3e410d3a..470121c1c 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/CatalogTaskScheduleJob.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/CatalogTaskScheduleJob.java @@ -19,8 +19,9 @@ import io.datavines.common.utils.DateUtils; import io.datavines.core.constant.DataVinesConstants; import io.datavines.server.api.dto.bo.catalog.CatalogRefresh; +import io.datavines.server.enums.CommonTaskType; import io.datavines.server.repository.entity.DataSource; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskService; +import io.datavines.server.repository.service.CommonTaskService; import io.datavines.server.repository.service.impl.JobExternalService; import io.datavines.server.utils.SpringApplicationContext; import org.quartz.JobDataMap; @@ -59,9 +60,13 @@ public void execute(JobExecutionContext context) { logger.info("scheduled fire time :{}, fire time :{}, dataSource id :{}", scheduleTime, fireTime, dataSourceId); logger.info("scheduled start work , dataSource id :{} ", dataSourceId); - CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService = getJobExternalService().getCatalogTaskService(); + CommonTaskService commonTaskService = getJobExternalService().getCatalogTaskService(); CatalogRefresh catalogRefresh = new CatalogRefresh(); catalogRefresh.setDatasourceId(dataSourceId); + if (dataMap.get(DataVinesConstants.COMMON_TASK_TYPE) != null) { + Integer taskType = dataMap.getInt(DataVinesConstants.COMMON_TASK_TYPE); + catalogRefresh.setTaskType(CommonTaskType.of(taskType)); + } DataSource dataSource = getJobExternalService().getDataSourceService().getDataSourceById(dataSourceId); if (dataSource == null) { @@ -69,7 +74,7 @@ public void execute(JobExecutionContext context) { return; } - catalogMetaDataFetchTaskService.refreshCatalog(catalogRefresh); + commonTaskService.refreshCatalog(catalogRefresh); } } diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/QuartzExecutors.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/QuartzExecutors.java index 2d043b929..d6fbaae2e 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/QuartzExecutors.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/QuartzExecutors.java @@ -218,10 +218,13 @@ private static String buildJobGroupName(ScheduleJobInfo schedule) { * @param schedule schedule * @return map */ - private static Map buildDataMap( ScheduleJobInfo schedule) { + private static Map buildDataMap(ScheduleJobInfo schedule) { Map dataMap = Maps.newHashMap(); dataMap.put(DataVinesConstants.JOB_ID, schedule.getId()); dataMap.put(DataVinesConstants.DATASOURCE_ID, schedule.getDatasourceId()); + if (schedule.getTaskType() != null) { + dataMap.put(DataVinesConstants.COMMON_TASK_TYPE, schedule.getTaskType().getCode()); + } dataMap.put(DataVinesConstants.SCHEDULE, JSONUtils.toJsonString(schedule)); return dataMap; } diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/ScheduleJobInfo.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/ScheduleJobInfo.java index 4f9558942..708e70978 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/ScheduleJobInfo.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/quartz/ScheduleJobInfo.java @@ -16,6 +16,7 @@ */ package io.datavines.server.dqc.coordinator.quartz; +import io.datavines.server.enums.CommonTaskType; import io.datavines.server.enums.ScheduleJobType; import lombok.Data; @@ -26,6 +27,8 @@ public class ScheduleJobInfo { private ScheduleJobType type; + private CommonTaskType taskType; + private Long datasourceId; private Long id; @@ -37,12 +40,14 @@ public class ScheduleJobInfo { private LocalDateTime endTime; public ScheduleJobInfo(ScheduleJobType type, + CommonTaskType taskType, Long datasourceId, Long id, String cronExpression, LocalDateTime startTime, LocalDateTime endTime) { this.type = type; + this.taskType = taskType; this.datasourceId = datasourceId; this.id = id; this.cronExpression = cronExpression; diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java similarity index 97% rename from datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java rename to datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java index 357d85f5c..2614a6014 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/operator/JobResultValidator.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.dqc.coordinator.operator; +package io.datavines.server.dqc.coordinator.validator; import io.datavines.common.entity.JobExecutionRequest; import io.datavines.common.enums.OperatorType; @@ -110,9 +110,11 @@ private boolean checkDqExecuteResult(JobExecutionResult jobExecutionResult) { MetricExecutionResult metricExecutionResult = new MetricExecutionResult(); BeanUtils.copyProperties(jobExecutionResult, metricExecutionResult); if (MetricValidator.isSuccess(metricExecutionResult)) { + jobExecutionResult.setScore(MetricValidator.getQualityScore(metricExecutionResult, true)); jobExecutionResult.setState(DqJobExecutionState.SUCCESS.getCode()); result = true; } else { + jobExecutionResult.setScore(MetricValidator.getQualityScore(metricExecutionResult, false)); jobExecutionResult.setState(DqJobExecutionState.FAILURE.getCode()); } diff --git a/datavines-server/src/main/java/io/datavines/server/enums/CommonTaskType.java b/datavines-server/src/main/java/io/datavines/server/enums/CommonTaskType.java new file mode 100644 index 000000000..4487df237 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/enums/CommonTaskType.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.enums; + +import com.baomidou.mybatisplus.annotation.EnumValue; + +import java.util.HashMap; +import java.util.Map; + +public enum CommonTaskType { + + /** + * 0 - datasource + * 1 - database + * 2 - table + */ + CATALOG_METADATA_FETCH(0, "catalog_metadata_fetch"), + DATA_QUALITY_REPORT(1, "data_quality_report") + ; + + CommonTaskType(int code, String description){ + this.code = code; + this.description = description; + } + + private final int code; + + @EnumValue + private final String description; + + private static final Map FETCH_TYPE_MAP = new HashMap<>(); + + static { + for (CommonTaskType fetchType : CommonTaskType.values()) { + FETCH_TYPE_MAP.put(fetchType.code,fetchType); + } + } + + public static CommonTaskType of(Integer status) { + if (FETCH_TYPE_MAP.containsKey(status)) { + return FETCH_TYPE_MAP.get(status); + } + throw new IllegalArgumentException("invalid fetch type : " + status); + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/datavines-server/src/main/java/io/datavines/server/enums/DataQualityLevel.java b/datavines-server/src/main/java/io/datavines/server/enums/DataQualityLevel.java new file mode 100644 index 000000000..a16d98a7d --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/enums/DataQualityLevel.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.enums; + +import com.baomidou.mybatisplus.annotation.EnumValue; +import io.datavines.connector.plugin.DatabendExecutor; + +import java.math.BigDecimal; + +public enum DataQualityLevel { + + UNQUALIFIED(1, "unqualified","不合格"), + QUALIFIED(2, "qualified","合格"), + MEDIUM(3, "medium","中等"), + GOOD(4, "good","良好"), + EXCELLENT(5, "excellent","优秀"); + + DataQualityLevel(int code, String description, String zhDescription) { + this.code = code; + this.description = description; + this.zhDescription = zhDescription; + } + + @EnumValue + private final int code; + + private final String description; + + private final String zhDescription; + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } + + public String getZhDescription() { + return zhDescription; + } + + public static DataQualityLevel getQualityLevelByScore(BigDecimal score) { + if ((score.compareTo(new BigDecimal(100)) == 0 || score.compareTo(new BigDecimal(100)) <= -1) + && (score.compareTo(new BigDecimal(80)) >= 0)) { + return EXCELLENT; + } else if ((score.compareTo(new BigDecimal(80)) == 0 || score.compareTo(new BigDecimal(80)) <= -1) + && (score.compareTo(new BigDecimal(60)) >= 0)) { + return GOOD; + } else if ((score.compareTo(new BigDecimal(60)) == 0 || score.compareTo(new BigDecimal(60)) <= -1) + && (score.compareTo(new BigDecimal(40)) >= 0)) { + return MEDIUM; + } else if ((score.compareTo(new BigDecimal(40)) == 0 || score.compareTo(new BigDecimal(40)) <= -1) + && (score.compareTo(new BigDecimal(20)) >= 0)) { + return QUALIFIED; + } else { + return UNQUALIFIED; + } + } +} diff --git a/datavines-server/src/main/java/io/datavines/server/registry/Register.java b/datavines-server/src/main/java/io/datavines/server/registry/Register.java index 7653e0e17..01c791cad 100644 --- a/datavines-server/src/main/java/io/datavines/server/registry/Register.java +++ b/datavines-server/src/main/java/io/datavines/server/registry/Register.java @@ -23,7 +23,7 @@ import io.datavines.registry.api.Registry; import io.datavines.registry.api.ServerInfo; import io.datavines.registry.api.SubscribeListener; -import io.datavines.server.catalog.metadata.CatalogMetaDataFetchTaskFailover; +import io.datavines.server.scheduler.CommonTaskFailover; import io.datavines.server.dqc.coordinator.failover.JobExecutionFailover; import io.datavines.server.repository.entity.Config; import io.datavines.server.repository.service.ConfigService; @@ -46,7 +46,7 @@ public class Register { private final JobExecutionFailover jobExecutionFailover; - private final CatalogMetaDataFetchTaskFailover catalogMetaDataFetchTaskFailover; + private final CommonTaskFailover commonTaskFailover; private final ConfigService configService; @@ -65,10 +65,10 @@ public class Register { private final String FAILOVER_KEY = CommonPropertyUtils.getString(CommonPropertyUtils.FAILOVER_KEY, CommonPropertyUtils.FAILOVER_KEY_DEFAULT); - public Register(Registry registry, JobExecutionFailover jobExecutionFailover, CatalogMetaDataFetchTaskFailover catalogMetaDataFetchTaskFailover) { + public Register(Registry registry, JobExecutionFailover jobExecutionFailover, CommonTaskFailover commonTaskFailover) { this.registry = registry; this.jobExecutionFailover = jobExecutionFailover; - this.catalogMetaDataFetchTaskFailover = catalogMetaDataFetchTaskFailover; + this.commonTaskFailover = commonTaskFailover; this.configService = SpringApplicationContext.getBean(ConfigService.class); updateCommonProperties(); } @@ -95,7 +95,7 @@ public void start() { try { blockUtilAcquireLock(FAILOVER_KEY); jobExecutionFailover.handleJobExecutionFailover(event.key()); - catalogMetaDataFetchTaskFailover.handleMetaDataFetchTaskFailover(event.key()); + commonTaskFailover.handleMetaDataFetchTaskFailover(event.key()); } finally { registry.release(FAILOVER_KEY); } @@ -113,7 +113,7 @@ public void start() { String host = NetUtils.getAddr(CommonPropertyUtils.getInt( CommonPropertyUtils.SERVER_PORT, CommonPropertyUtils.SERVER_PORT_DEFAULT)); jobExecutionFailover.handleJobExecutionFailover(host); - catalogMetaDataFetchTaskFailover.handleMetaDataFetchTaskFailover(host); + commonTaskFailover.handleMetaDataFetchTaskFailover(host); List activeServerInfoList = registry.getActiveServerList(); //Get the current active server, and then get all running tasks of the server other than the active server list @@ -123,7 +123,7 @@ public void start() { .collect(Collectors.toList()); jobExecutionFailover.handleJobExecutionFailover(activeServerList); - catalogMetaDataFetchTaskFailover.handleMetaDataFetchTaskFailover(activeServerList); + commonTaskFailover.handleMetaDataFetchTaskFailover(activeServerList); } finally { registry.release(FAILOVER_KEY); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/ActualValues.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/ActualValues.java index e9d27d2fb..d2da3fa48 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/entity/ActualValues.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/ActualValues.java @@ -24,6 +24,7 @@ import lombok.Data; import java.io.Serializable; +import java.math.BigDecimal; import java.time.LocalDateTime; @Data @@ -48,7 +49,7 @@ public class ActualValues implements Serializable { private String actualName; @TableField(value = "actual_value") - private Double actualValue; + private BigDecimal actualValue; @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") @TableField(value = "data_time") diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTask.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTask.java similarity index 91% rename from datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTask.java rename to datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTask.java index 99c4a352c..9f1a672a1 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTask.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTask.java @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.repository.entity.catalog; +package io.datavines.server.repository.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; +import io.datavines.server.enums.CommonTaskType; import io.datavines.server.enums.FetchType; import lombok.Data; @@ -28,14 +29,17 @@ import java.time.LocalDateTime; @Data -@TableName("dv_catalog_metadata_fetch_task") -public class CatalogMetaDataFetchTask implements Serializable { +@TableName("dv_common_task") +public class CommonTask implements Serializable { private static final long serialVersionUID = -1L; @TableId(type= IdType.AUTO) private Long id; + @TableField(value = "task_type") + private CommonTaskType taskType; + @TableField(value = "type") private FetchType type; diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchCommand.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskCommand.java similarity index 90% rename from datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchCommand.java rename to datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskCommand.java index b006b0be1..544bfbc86 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchCommand.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskCommand.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.repository.entity.catalog; +package io.datavines.server.repository.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -28,8 +28,8 @@ import java.time.LocalDateTime; @Data -@TableName("dv_catalog_metadata_fetch_command") -public class CatalogMetaDataFetchCommand implements Serializable { +@TableName("dv_common_task_command") +public class CommonTaskCommand implements Serializable { private static final long serialVersionUID = -1L; diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTaskSchedule.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskSchedule.java similarity index 89% rename from datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTaskSchedule.java rename to datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskSchedule.java index f3fc285b3..ad2344583 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/entity/catalog/CatalogMetaDataFetchTaskSchedule.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/CommonTaskSchedule.java @@ -14,21 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.repository.entity.catalog; +package io.datavines.server.repository.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; +import io.datavines.server.enums.CommonTaskType; import lombok.Data; import java.io.Serializable; import java.time.LocalDateTime; @Data -@TableName("dv_catalog_metadata_fetch_task_schedule") -public class CatalogMetaDataFetchTaskSchedule implements Serializable { +@TableName("dv_common_task_schedule") +public class CommonTaskSchedule implements Serializable { private static final long serialVersionUID = -1L; @@ -38,6 +39,9 @@ public class CatalogMetaDataFetchTaskSchedule implements Serializable { @TableField(value = "datasource_id") private Long dataSourceId; + @TableField(value = "task_type") + private CommonTaskType taskType; + @TableField(value = "type") private String type; diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResult.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResult.java index fc34e4a7e..66fafb541 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResult.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResult.java @@ -24,6 +24,7 @@ import lombok.Data; import java.io.Serializable; +import java.math.BigDecimal; import java.time.LocalDateTime; @Data @@ -60,10 +61,10 @@ public class JobExecutionResult implements Serializable { private String columnName; @TableField(value = "actual_value") - private Double actualValue; + private BigDecimal actualValue; @TableField(value = "expected_value") - private Double expectedValue; + private BigDecimal expectedValue; @TableField(value = "expected_type") private String expectedType; @@ -75,7 +76,10 @@ public class JobExecutionResult implements Serializable { private String operator; @TableField(value = "threshold") - private Double threshold; + private BigDecimal threshold; + + @TableField(value = "score") + private BigDecimal score; @TableField(value = "state") private int state; diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResultReportRel.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResultReportRel.java new file mode 100644 index 000000000..1a4e08c9c --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobExecutionResultReportRel.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; + +import java.io.Serializable; +import java.time.LocalDateTime; + +@Data +@TableName("dv_job_execution_result_report_rel") +public class JobExecutionResultReportRel implements Serializable { + + private static final long serialVersionUID = -1L; + + @TableId(type = IdType.AUTO) + private Long id; + + @TableField(value = "job_execution_result_id") + private Long jobExecutionResultId; + + @TableField(value = "quality_report_id") + private Long qualityReportId; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @TableField(value = "create_time") + private LocalDateTime createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @TableField(value = "update_time") + private LocalDateTime updateTime; +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/entity/JobQualityReport.java b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobQualityReport.java new file mode 100644 index 000000000..cec7804e7 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/entity/JobQualityReport.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; + +@Data +@TableName("dv_job_quality_report") +public class JobQualityReport implements Serializable { + + private static final long serialVersionUID = -1L; + + @TableId(type= IdType.AUTO) + private Long id; + + @TableField(value = "datasource_id") + private Long datasourceId; + + @TableField(value = "entity_level") + private String entityLevel; + + @TableField(value = "database_name") + private String databaseName; + + @TableField(value = "table_name") + private String tableName; + + @TableField(value = "column_name") + private String columnName; + + @TableField(value = "score") + private BigDecimal score; + + @TableField(value = "report_date") + private LocalDate reportDate; + + @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @TableField(value = "create_time") + private LocalDateTime createTime; + + @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") + @TableField(value = "update_time") + private LocalDateTime updateTime; +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchCommandMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskCommandMapper.java similarity index 70% rename from datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchCommandMapper.java rename to datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskCommandMapper.java index 0ff71285e..9cbf09033 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchCommandMapper.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskCommandMapper.java @@ -17,14 +17,14 @@ package io.datavines.server.repository.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; +import io.datavines.server.repository.entity.CommonTaskCommand; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @Mapper -public interface CatalogMetaDataFetchCommandMapper extends BaseMapper { +public interface CommonTaskCommandMapper extends BaseMapper { - @Select("SELECT * from dv_catalog_metadata_fetch_command where id % #{totalSlot} = #{currentSlot} order by update_time limit 1 ") - CatalogMetaDataFetchCommand getOne(@Param("totalSlot") int totalSlot, @Param("currentSlot") int currentSlot); + @Select("SELECT * from dv_common_task_command where id % #{totalSlot} = #{currentSlot} order by update_time limit 1 ") + CommonTaskCommand getOne(@Param("totalSlot") int totalSlot, @Param("currentSlot") int currentSlot); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskMapper.java similarity index 86% rename from datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskMapper.java rename to datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskMapper.java index 64020c27c..bff60105e 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskMapper.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskMapper.java @@ -21,13 +21,13 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.datavines.server.api.dto.vo.catalog.CatalogMetaDataFetchTaskVO; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; +import io.datavines.server.repository.entity.CommonTask; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @Mapper -public interface CatalogMetaDataFetchTaskMapper extends BaseMapper { +public interface CommonTaskMapper extends BaseMapper { IPage getJobExecutionPage(Page page, - @Param("datasourceId") Long datasourceId); + @Param("datasourceId") Long datasourceId, @Param("taskType") String taskType); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskScheduleMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskScheduleMapper.java new file mode 100644 index 000000000..a04a31d65 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/CommonTaskScheduleMapper.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.datavines.server.repository.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import io.datavines.server.repository.entity.CommonTaskSchedule; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +@Mapper +public interface CommonTaskScheduleMapper extends BaseMapper { + + @Select("SELECT * from dv_common_task_schedule WHERE datasource_id = #{datasourceId} and task_type = #{taskType} limit 1") + CommonTaskSchedule getByDataSourceIdAndType(@Param("datasourceId") long datasourceId , @Param("taskType") String taskType); + + @Select("SELECT * from dv_common_task_schedule WHERE datasource_id = #{datasourceId} limit 1") + CommonTaskSchedule getByDataSourceId(long datasourceId); +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionMapper.java index ad8d35e66..5dc196100 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionMapper.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionMapper.java @@ -52,11 +52,11 @@ List getJobExecutionAggPie(@Param("datasourceId") Long data @Param("metricType") String metricType, @Param("schemaName") String schemaName, @Param("tableName") String tableName, @Param("columnName") String columnName, @Param("startTime") String startTime, @Param("endTime") String endTime); + List getJobExecutionTrendBar(@Param("datasourceId") Long datasourceId, @Param("metricType") String metricType, @Param("schemaName") String schemaName, @Param("tableName") String tableName, @Param("columnName") String columnName, @Param("startTime") String startTime, @Param("endTime") String endTime); - JobExecutionStat getJobExecutionStat(@Param("jobId") Long jobId); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultMapper.java index 85676ef4c..fd9203d14 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultMapper.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultMapper.java @@ -35,6 +35,16 @@ public interface JobExecutionResultMapper extends BaseMapper List listByJobIdAndTimeRange(@Param("jobId") Long jobId, @Param("startTime")String startTime, @Param("endTime")String endTime); + @Select("SELECT t1.id,t1.metric_name,t1.database_name,t1.table_name,t1.column_name,t1.score\n" + + "FROM\n" + + "\t(select djer.* from dv_job_execution_result djer join dv_job_execution dje on djer.job_execution_id = dje.id where dje.datasource_id = #{datasourceId} and djer.create_time >= #{startTime} and djer.create_time <= #{endTime}\n" + + ") t1\n" + + "\tINNER JOIN ( SELECT t3.database_name,t3.table_name,t3.column_name,t3.metric_name, MAX(t3.create_time) as max_timestamp FROM (select djer.* from dv_job_execution_result djer join dv_job_execution dje on djer.job_execution_id = dje.id where dje.datasource_id = #{datasourceId}\n" + + ") t3 GROUP BY database_name,table_name,column_name,metric_name ) t2\n" + + " ON t1.database_name = t2.database_name and t1.table_name = t2.table_name and t1.column_name = t2.column_name and t1.metric_name = t2.metric_name and t1.create_time = t2.max_timestamp;") + List listByDatasourceIdAndTimeRange(@Param("datasourceId") Long datasourceId, + @Param("startTime")String startTime, + @Param("endTime")String endTime); List listByJobExecutionId(@Param("jobExecutionIdList") List jobExecutionIdList); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskScheduleMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultReportRelMapper.java similarity index 62% rename from datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskScheduleMapper.java rename to datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultReportRelMapper.java index e7967a89f..95880d393 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/mapper/CatalogMetaDataFetchTaskScheduleMapper.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobExecutionResultReportRelMapper.java @@ -18,14 +18,17 @@ package io.datavines.server.repository.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTaskSchedule; +import io.datavines.server.repository.entity.JobExecutionResult; +import io.datavines.server.repository.entity.JobExecutionResultReportRel; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; -@Mapper -public interface CatalogMetaDataFetchTaskScheduleMapper extends BaseMapper { +import java.util.List; - @Select("SELECT * from dv_catalog_metadata_fetch_task_schedule WHERE datasource_id = #{datasourceId} limit 1") - CatalogMetaDataFetchTaskSchedule getByDataSourceId(long datasourceId); +@Mapper +public interface JobExecutionResultReportRelMapper extends BaseMapper { + @Select("SELECT djer.* from dv_job_execution_result djer join dv_job_execution_result_report_rel djerrr on djer.id = djerrr.job_execution_result_id where djerrr.quality_report_id = #{reportId}") + List listExecutionResultByReportId(@Param("reportId") Long reportId); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobQualityReportMapper.java b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobQualityReportMapper.java new file mode 100644 index 000000000..ff26b5afe --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/mapper/JobQualityReportMapper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import io.datavines.server.repository.entity.JobQualityReport; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +@Mapper +public interface JobQualityReportMapper extends BaseMapper { + + @Select("SELECT datasource_id, database_name, table_name, '--' as column_name, avg(score) as score, #{reportDate} as report_date, 'table' as entity_level" + + " from dv_job_quality_report where report_date = #{reportDate} and datasource_id = #{datasourceId} and entity_level = 'column'" + + " group by datasource_id, database_name, table_name") + List listTableScoreGroupByDatasource(@Param("datasourceId") Long datasourceId, + @Param("reportDate") String reportDate); + + @Select("SELECT datasource_id, database_name, '--' as table_name, '--' as column_name, avg(score) as score, #{reportDate} as report_date, 'database' as entity_level" + + " from dv_job_quality_report where report_date = #{reportDate} and datasource_id = #{datasourceId} and entity_level = 'table'" + + " group by datasource_id, database_name") + List listDbScoreGroupByDatasource(@Param("datasourceId") Long datasourceId, + @Param("reportDate") String reportDate); + + @Select("SELECT datasource_id, '--' as database_name, '--' as table_name, '--' as column_name, avg(score) as score, #{reportDate} as report_date, 'datasource' as entity_level" + + " from dv_job_quality_report where report_date = #{reportDate} and datasource_id = #{datasourceId} and entity_level = 'database'" + + " group by datasource_id") + List listDatasourceScoreGroupByDatasource(@Param("datasourceId") Long datasourceId, + @Param("reportDate") String reportDate); +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchCommandService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskCommandService.java similarity index 67% rename from datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchCommandService.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskCommandService.java index 1d2e2d6cd..4572d85a6 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchCommandService.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskCommandService.java @@ -17,17 +17,17 @@ package io.datavines.server.repository.service; import com.baomidou.mybatisplus.extension.service.IService; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; +import io.datavines.server.repository.entity.CommonTaskCommand; -public interface CatalogMetaDataFetchCommandService extends IService { +public interface CommonTaskCommandService extends IService { - long create(CatalogMetaDataFetchCommand catalogMetaDataFetchCommand); + long create(CommonTaskCommand commonTaskCommand); - int update(CatalogMetaDataFetchCommand catalogMetaDataFetchCommand); + int update(CommonTaskCommand commonTaskCommand); - CatalogMetaDataFetchCommand getById(long id); + CommonTaskCommand getById(long id); - CatalogMetaDataFetchCommand getOne(int totalSlot, int currentSlot); + CommonTaskCommand getOne(int totalSlot, int currentSlot); int deleteById(long id); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskScheduleService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskScheduleService.java similarity index 67% rename from datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskScheduleService.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskScheduleService.java index 212872e26..ba1921906 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskScheduleService.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskScheduleService.java @@ -19,23 +19,23 @@ import com.baomidou.mybatisplus.extension.service.IService; import io.datavines.core.exception.DataVinesServerException; -import io.datavines.server.api.dto.bo.catalog.metadata.CatalogMetaDataFetchTaskScheduleCreateOrUpdate; +import io.datavines.server.api.dto.bo.task.CommonTaskScheduleCreateOrUpdate; import io.datavines.server.api.dto.bo.job.schedule.MapParam; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTaskSchedule; +import io.datavines.server.repository.entity.CommonTaskSchedule; import java.util.List; -public interface CatalogMetaDataFetchTaskScheduleService extends IService { +public interface CommonTaskScheduleService extends IService { - CatalogMetaDataFetchTaskSchedule createOrUpdate(CatalogMetaDataFetchTaskScheduleCreateOrUpdate scheduleCreate) throws DataVinesServerException; + CommonTaskSchedule createOrUpdate(CommonTaskScheduleCreateOrUpdate scheduleCreate) throws DataVinesServerException; boolean deleteById(long id); boolean deleteByDataSourceId(long datasourceId); - CatalogMetaDataFetchTaskSchedule getById(long id); + CommonTaskSchedule getById(long id); - CatalogMetaDataFetchTaskSchedule getByDataSourceId(Long dataSourceId); + CommonTaskSchedule getByDataSourceId(Long dataSourceId, String taskType); List getCron(MapParam mapParam); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskService.java similarity index 76% rename from datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskService.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskService.java index a865b6813..e2ac1d725 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/CatalogMetaDataFetchTaskService.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/CommonTaskService.java @@ -20,24 +20,24 @@ import com.baomidou.mybatisplus.extension.service.IService; import io.datavines.server.api.dto.bo.catalog.CatalogRefresh; import io.datavines.server.api.dto.vo.catalog.CatalogMetaDataFetchTaskVO; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; +import io.datavines.server.repository.entity.CommonTask; import java.time.LocalDateTime; import java.util.List; -public interface CatalogMetaDataFetchTaskService extends IService { +public interface CommonTaskService extends IService { long refreshCatalog(CatalogRefresh catalogRefresh); - int update(CatalogMetaDataFetchTask catalogMetaDataFetchTask); + int update(CommonTask commonTask); - CatalogMetaDataFetchTask getById(long id); + CommonTask getById(long id); Long killCatalogTask(Long catalogTaskId); - List listNeedFailover(String host); + List listNeedFailover(String host); - List listTaskNotInServerList(List hostList); + List listTaskNotInServerList(List hostList); String getTaskExecuteHost(Long catalogTaskId); @@ -45,5 +45,5 @@ public interface CatalogMetaDataFetchTaskService extends IService getFetchTaskPage(Long datasourceId,Integer pageNumber, Integer pageSize); + IPage getFetchTaskPage(Long datasourceId, String taskType, Integer pageNumber, Integer pageSize); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/JobExecutionResultReportRelService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/JobExecutionResultReportRelService.java new file mode 100644 index 000000000..598ab46d9 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/JobExecutionResultReportRelService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import io.datavines.server.repository.entity.JobExecutionResult; +import io.datavines.server.repository.entity.JobExecutionResultReportRel; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface JobExecutionResultReportRelService extends IService { + + List listExecutionResultByReportId(Long reportId); +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/JobQualityReportService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/JobQualityReportService.java new file mode 100644 index 000000000..05407f73d --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/JobQualityReportService.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.datavines.server.api.dto.bo.job.JobQualityReportDashboardParam; +import io.datavines.server.api.dto.vo.JobExecutionResultVO; +import io.datavines.server.api.dto.vo.JobQualityReportScore; +import io.datavines.server.api.dto.vo.JobQualityReportScoreTrend; +import io.datavines.server.api.dto.vo.JobQualityReportVO; +import io.datavines.server.repository.entity.JobExecutionResult; +import io.datavines.server.repository.entity.JobQualityReport; + +import java.util.List; + +public interface JobQualityReportService extends IService { + + IPage getQualityReportPage(JobQualityReportDashboardParam dashboardParam); + + boolean generateQualityReport(Long datasourceId); + + JobQualityReportScore getScoreByCondition(JobQualityReportDashboardParam dashboardParam); + + JobQualityReportScoreTrend getScoreTrendByCondition(JobQualityReportDashboardParam dashboardParam); + + List listColumnExecution(Long reportId); +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java index 8c9b82a49..458340f3e 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java @@ -26,7 +26,6 @@ import io.datavines.common.entity.job.BaseJobParameter; import io.datavines.common.enums.DataVinesDataType; import io.datavines.common.enums.EntityRelType; -import io.datavines.common.enums.JobType; import io.datavines.common.utils.CommonPropertyUtils; import io.datavines.common.utils.DateUtils; import io.datavines.common.utils.JSONUtils; @@ -54,7 +53,6 @@ import java.text.DecimalFormat; import java.time.LocalDateTime; import java.util.*; -import java.util.function.Function; import java.util.stream.Collectors; import static io.datavines.common.enums.JobType.DATA_PROFILE; @@ -83,7 +81,7 @@ public class CatalogEntityInstanceServiceImpl private CatalogEntityProfileService catalogEntityProfileService; @Autowired - private CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService; + private CommonTaskService commonTaskService; @Override public String create(CatalogEntityInstance entityInstance) { @@ -333,7 +331,7 @@ public CatalogDatabaseDetailVO getDatabaseEntityDetail(String uuid) { detail.setName(databaseInstance.getDisplayName()); detail.setType(databaseInstance.getType()); detail.setUuid(uuid); - detail.setUpdateTime(catalogMetaDataFetchTaskService.getRefreshTime(databaseInstance.getDatasourceId(), databaseInstance.getDisplayName(),null)); + detail.setUpdateTime(commonTaskService.getRefreshTime(databaseInstance.getDatasourceId(), databaseInstance.getDisplayName(),null)); List tableList = getCatalogEntityInstances(uuid); detail.setTables((long)(CollectionUtils.isEmpty(tableList)? 0 : tableList.size())); detail.setMetrics(getEntityMetricCount(uuid)); @@ -360,7 +358,7 @@ public CatalogTableDetailVO getTableEntityDetail(String uuid) { if (values.length == 2) { database = values[0]; table = values[1]; - detail.setUpdateTime(catalogMetaDataFetchTaskService.getRefreshTime(instance.getDatasourceId(), database,table)); + detail.setUpdateTime(commonTaskService.getRefreshTime(instance.getDatasourceId(), database,table)); } else { detail.setUpdateTime(instance.getUpdateTime()); } @@ -396,7 +394,7 @@ public CatalogColumnDetailVO getColumnEntityDetail(String uuid) { if (values.length == 3) { database = values[0]; table = values[1]; - detail.setUpdateTime(catalogMetaDataFetchTaskService.getRefreshTime(instance.getDatasourceId(), database,table)); + detail.setUpdateTime(commonTaskService.getRefreshTime(instance.getDatasourceId(), database,table)); } else { detail.setUpdateTime(instance.getUpdateTime()); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityProfileServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityProfileServiceImpl.java index d68a8ce50..29c3cf8a1 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityProfileServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityProfileServiceImpl.java @@ -95,9 +95,9 @@ public List listTableRecords(String uuid, String starTime, S .le(CatalogEntityProfile::getDataDate, endTime) .orderByAsc(CatalogEntityProfile::getDataDate)); - return tableRowCounts.stream().map(item -> { - return new DataTime2ValueItem(item.getDataDate(), item.getActualValue()); - }).collect(Collectors.toList()); + return tableRowCounts.stream() + .map(item -> new DataTime2ValueItem(item.getDataDate(), item.getActualValue())) + .collect(Collectors.toList()); } @Override diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchCommandServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskCommandServiceImpl.java similarity index 58% rename from datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchCommandServiceImpl.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskCommandServiceImpl.java index 19b72a3ef..0a4a94d20 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchCommandServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskCommandServiceImpl.java @@ -17,35 +17,35 @@ package io.datavines.server.repository.service.impl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; -import io.datavines.server.repository.mapper.CatalogMetaDataFetchCommandMapper; -import io.datavines.server.repository.service.CatalogMetaDataFetchCommandService; +import io.datavines.server.repository.entity.CommonTaskCommand; +import io.datavines.server.repository.mapper.CommonTaskCommandMapper; +import io.datavines.server.repository.service.CommonTaskCommandService; import org.springframework.stereotype.Service; -@Service("catalogCommandService") -public class CatalogMetaDataFetchCommandServiceImpl - extends ServiceImpl - implements CatalogMetaDataFetchCommandService { +@Service("ccommonTaskCommandService") +public class CommonTaskCommandServiceImpl + extends ServiceImpl + implements CommonTaskCommandService { @Override - public long create(CatalogMetaDataFetchCommand catalogMetaDataFetchCommand) { - baseMapper.insert(catalogMetaDataFetchCommand); - return catalogMetaDataFetchCommand.getId(); + public long create(CommonTaskCommand commonTaskCommand) { + baseMapper.insert(commonTaskCommand); + return commonTaskCommand.getId(); } @Override - public int update(CatalogMetaDataFetchCommand catalogMetaDataFetchCommand) { + public int update(CommonTaskCommand commonTaskCommand) { return 0; } @Override - public CatalogMetaDataFetchCommand getById(long id) { + public CommonTaskCommand getById(long id) { return null; } @Override - public CatalogMetaDataFetchCommand getOne(int totalSlot, int currentSlot) { + public CommonTaskCommand getOne(int totalSlot, int currentSlot) { return baseMapper.getOne(totalSlot, currentSlot); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskScheduleServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskScheduleServiceImpl.java similarity index 53% rename from datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskScheduleServiceImpl.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskScheduleServiceImpl.java index c5aadbb18..52f423370 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskScheduleServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskScheduleServiceImpl.java @@ -22,7 +22,7 @@ import io.datavines.common.utils.JSONUtils; import io.datavines.core.enums.Status; import io.datavines.core.exception.DataVinesServerException; -import io.datavines.server.api.dto.bo.catalog.metadata.CatalogMetaDataFetchTaskScheduleCreateOrUpdate; +import io.datavines.server.api.dto.bo.task.CommonTaskScheduleCreateOrUpdate; import io.datavines.server.api.dto.bo.job.schedule.MapParam; import io.datavines.server.dqc.coordinator.quartz.CatalogTaskScheduleJob; import io.datavines.server.dqc.coordinator.quartz.QuartzExecutors; @@ -32,9 +32,9 @@ import io.datavines.server.enums.JobScheduleType; import io.datavines.server.enums.ScheduleJobType; import io.datavines.server.repository.entity.DataSource; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTaskSchedule; -import io.datavines.server.repository.mapper.CatalogMetaDataFetchTaskScheduleMapper; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskScheduleService; +import io.datavines.server.repository.entity.CommonTaskSchedule; +import io.datavines.server.repository.mapper.CommonTaskScheduleMapper; +import io.datavines.server.repository.service.CommonTaskScheduleService; import io.datavines.server.repository.service.DataSourceService; import io.datavines.server.utils.ContextHolder; import lombok.extern.slf4j.Slf4j; @@ -49,8 +49,8 @@ import java.util.List; @Slf4j -@Service("catalogMetaDataFetchTaskScheduleService") -public class CatalogMetaDataFetchTaskScheduleServiceImpl extends ServiceImpl implements CatalogMetaDataFetchTaskScheduleService { +@Service("commonTaskScheduleService") +public class CommonTaskScheduleServiceImpl extends ServiceImpl implements CommonTaskScheduleService { @Autowired private QuartzExecutors quartzExecutor; @@ -60,7 +60,7 @@ public class CatalogMetaDataFetchTaskScheduleServiceImpl extends ServiceImpl().lambda().eq(CatalogMetaDataFetchTaskSchedule::getDataSourceId, dataSourceId)); - if (catalogMetaDataFetchTaskSchedule != null) { - throw new DataVinesServerException(Status.CATALOG_TASK_SCHEDULE_EXIST_ERROR, catalogMetaDataFetchTaskSchedule.getId()); + CommonTaskSchedule commonTaskSchedule = baseMapper.selectOne(new QueryWrapper().lambda().eq(CommonTaskSchedule::getDataSourceId, dataSourceId)); + if (commonTaskSchedule != null) { + throw new DataVinesServerException(Status.CATALOG_TASK_SCHEDULE_EXIST_ERROR, commonTaskSchedule.getId()); } - catalogMetaDataFetchTaskSchedule = new CatalogMetaDataFetchTaskSchedule(); - BeanUtils.copyProperties(scheduleCreateOrUpdate, catalogMetaDataFetchTaskSchedule); - catalogMetaDataFetchTaskSchedule.setCreateBy(ContextHolder.getUserId()); - catalogMetaDataFetchTaskSchedule.setCreateTime(LocalDateTime.now()); - catalogMetaDataFetchTaskSchedule.setUpdateBy(ContextHolder.getUserId()); - catalogMetaDataFetchTaskSchedule.setUpdateTime(LocalDateTime.now()); - catalogMetaDataFetchTaskSchedule.setStatus(true); + commonTaskSchedule = new CommonTaskSchedule(); + BeanUtils.copyProperties(scheduleCreateOrUpdate, commonTaskSchedule); + commonTaskSchedule.setCreateBy(ContextHolder.getUserId()); + commonTaskSchedule.setCreateTime(LocalDateTime.now()); + commonTaskSchedule.setUpdateBy(ContextHolder.getUserId()); + commonTaskSchedule.setUpdateTime(LocalDateTime.now()); + commonTaskSchedule.setStatus(true); - updateCatalogTaskScheduleParam(catalogMetaDataFetchTaskSchedule, scheduleCreateOrUpdate.getType(), scheduleCreateOrUpdate.getParam()); + updateCatalogTaskScheduleParam(commonTaskSchedule, scheduleCreateOrUpdate.getType(), scheduleCreateOrUpdate.getParam()); DataSource dataSource = dataSourceService.getById(dataSourceId); if (dataSource == null) { throw new DataVinesServerException(Status.DATASOURCE_NOT_EXIST_ERROR, dataSourceId); } else { - if (baseMapper.insert(catalogMetaDataFetchTaskSchedule) <= 0) { - log.info("create catalog task schedule fail : {}", catalogMetaDataFetchTaskSchedule); + if (baseMapper.insert(commonTaskSchedule) <= 0) { + log.info("create catalog task schedule fail : {}", commonTaskSchedule); throw new DataVinesServerException(Status.CREATE_CATALOG_TASK_SCHEDULE_ERROR); } try { - addScheduleJob(scheduleCreateOrUpdate, catalogMetaDataFetchTaskSchedule); + addScheduleJob(scheduleCreateOrUpdate, commonTaskSchedule); } catch (Exception e) { throw new DataVinesServerException(Status.ADD_QUARTZ_ERROR); } } log.info("create job schedule success: datasource id : {}, cronExpression : {}", - catalogMetaDataFetchTaskSchedule.getDataSourceId(), - catalogMetaDataFetchTaskSchedule.getCronExpression()); + commonTaskSchedule.getDataSourceId(), + commonTaskSchedule.getCronExpression()); - return catalogMetaDataFetchTaskSchedule; + return commonTaskSchedule; } - private void addScheduleJob(CatalogMetaDataFetchTaskScheduleCreateOrUpdate scheduleCreateOrUpdate, CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule) throws ParseException { + private void addScheduleJob(CommonTaskScheduleCreateOrUpdate scheduleCreateOrUpdate, CommonTaskSchedule commonTaskSchedule) throws ParseException { switch (JobScheduleType.of(scheduleCreateOrUpdate.getType())) { case CYCLE: case CRONTAB: - quartzExecutor.addJob(CatalogTaskScheduleJob.class, getScheduleJobInfo(catalogMetaDataFetchTaskSchedule)); + quartzExecutor.addJob(CatalogTaskScheduleJob.class, getScheduleJobInfo(commonTaskSchedule)); break; case OFFLINE: break; @@ -119,17 +119,17 @@ private void addScheduleJob(CatalogMetaDataFetchTaskScheduleCreateOrUpdate sched } } - private CatalogMetaDataFetchTaskSchedule update(CatalogMetaDataFetchTaskScheduleCreateOrUpdate scheduleCreateOrUpdate) throws DataVinesServerException { - CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule = getById(scheduleCreateOrUpdate.getId()); - if (catalogMetaDataFetchTaskSchedule == null) { + private CommonTaskSchedule update(CommonTaskScheduleCreateOrUpdate scheduleCreateOrUpdate) throws DataVinesServerException { + CommonTaskSchedule commonTaskSchedule = getById(scheduleCreateOrUpdate.getId()); + if (commonTaskSchedule == null) { throw new DataVinesServerException(Status.CATALOG_TASK_SCHEDULE_NOT_EXIST_ERROR, scheduleCreateOrUpdate.getId()); } - BeanUtils.copyProperties(scheduleCreateOrUpdate, catalogMetaDataFetchTaskSchedule); - catalogMetaDataFetchTaskSchedule.setUpdateBy(ContextHolder.getUserId()); - catalogMetaDataFetchTaskSchedule.setUpdateTime(LocalDateTime.now()); + BeanUtils.copyProperties(scheduleCreateOrUpdate, commonTaskSchedule); + commonTaskSchedule.setUpdateBy(ContextHolder.getUserId()); + commonTaskSchedule.setUpdateTime(LocalDateTime.now()); - updateCatalogTaskScheduleParam(catalogMetaDataFetchTaskSchedule, scheduleCreateOrUpdate.getType(), scheduleCreateOrUpdate.getParam()); + updateCatalogTaskScheduleParam(commonTaskSchedule, scheduleCreateOrUpdate.getType(), scheduleCreateOrUpdate.getParam()); Long dataSourceId = scheduleCreateOrUpdate.getDataSourceId(); if (dataSourceId == null) { @@ -137,27 +137,27 @@ private CatalogMetaDataFetchTaskSchedule update(CatalogMetaDataFetchTaskSchedule } try { - quartzExecutor.deleteJob(getScheduleJobInfo(catalogMetaDataFetchTaskSchedule)); - addScheduleJob(scheduleCreateOrUpdate, catalogMetaDataFetchTaskSchedule); + quartzExecutor.deleteJob(getScheduleJobInfo(commonTaskSchedule)); + addScheduleJob(scheduleCreateOrUpdate, commonTaskSchedule); } catch (Exception e) { throw new DataVinesServerException(Status.ADD_QUARTZ_ERROR); } - if (baseMapper.updateById(catalogMetaDataFetchTaskSchedule) <= 0) { - log.info("update catalog task schedule fail : {}", catalogMetaDataFetchTaskSchedule); - throw new DataVinesServerException(Status.UPDATE_CATALOG_TASK_SCHEDULE_ERROR, catalogMetaDataFetchTaskSchedule.getId()); + if (baseMapper.updateById(commonTaskSchedule) <= 0) { + log.info("update catalog task schedule fail : {}", commonTaskSchedule); + throw new DataVinesServerException(Status.UPDATE_CATALOG_TASK_SCHEDULE_ERROR, commonTaskSchedule.getId()); } - return catalogMetaDataFetchTaskSchedule; + return commonTaskSchedule; } @Override @Transactional(rollbackFor = Exception.class) public boolean deleteById(long id) { - CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule = getById(id); - if (catalogMetaDataFetchTaskSchedule != null) { - boolean deleteJob = quartzExecutor.deleteJob(getScheduleJobInfo(catalogMetaDataFetchTaskSchedule)); + CommonTaskSchedule commonTaskSchedule = getById(id); + if (commonTaskSchedule != null) { + boolean deleteJob = quartzExecutor.deleteJob(getScheduleJobInfo(commonTaskSchedule)); if (!deleteJob) { return false; } @@ -170,42 +170,42 @@ public boolean deleteById(long id) { @Override @Transactional(rollbackFor = Exception.class) public boolean deleteByDataSourceId(long dataSourceId) { - CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule = baseMapper.getByDataSourceId(dataSourceId); - if (catalogMetaDataFetchTaskSchedule == null) { + CommonTaskSchedule commonTaskSchedule = baseMapper.getByDataSourceId(dataSourceId); + if (commonTaskSchedule == null) { return false; } - boolean deleteJob = quartzExecutor.deleteJob(getScheduleJobInfo(catalogMetaDataFetchTaskSchedule)); + boolean deleteJob = quartzExecutor.deleteJob(getScheduleJobInfo(commonTaskSchedule)); if (!deleteJob ) { return false; } - removeById(catalogMetaDataFetchTaskSchedule.getId()); + removeById(commonTaskSchedule.getId()); return true; } @Override - public CatalogMetaDataFetchTaskSchedule getByDataSourceId(Long dataSourceId) { - return baseMapper.getByDataSourceId(dataSourceId); + public CommonTaskSchedule getByDataSourceId(Long dataSourceId, String taskType) { + return baseMapper.getByDataSourceIdAndType(dataSourceId, taskType); } @Override - public CatalogMetaDataFetchTaskSchedule getById(long id) { + public CommonTaskSchedule getById(long id) { return baseMapper.selectById(id); } @Override public List getCron(MapParam mapParam){ - List listCron = new ArrayList(); + List listCron = new ArrayList<>(); FunCron api = StrategyFactory.getByType(mapParam.getCycle()); - CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule = new CatalogMetaDataFetchTaskSchedule(); + CommonTaskSchedule commonTaskSchedule = new CommonTaskSchedule(); String result1 = JSONUtils.toJsonString(mapParam); - catalogMetaDataFetchTaskSchedule.setParam(result1); - String cron = api.funcDeal(catalogMetaDataFetchTaskSchedule.getParam()); + commonTaskSchedule.setParam(result1); + String cron = api.funcDeal(commonTaskSchedule.getParam()); listCron.add(cron); return listCron; } - private void updateCatalogTaskScheduleParam(CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule, String type, MapParam param) { + private void updateCatalogTaskScheduleParam(CommonTaskSchedule commonTaskSchedule, String type, MapParam param) { String paramStr = JSONUtils.toJsonString(param); switch (JobScheduleType.of(type)){ case CYCLE: @@ -216,10 +216,10 @@ private void updateCatalogTaskScheduleParam(CatalogMetaDataFetchTaskSchedule cat if (param.getCycle() == null) { throw new DataVinesServerException(Status.SCHEDULE_PARAMETER_IS_NULL_ERROR); } - catalogMetaDataFetchTaskSchedule.setStatus(true); - catalogMetaDataFetchTaskSchedule.setParam(paramStr); + commonTaskSchedule.setStatus(true); + commonTaskSchedule.setParam(paramStr); FunCron api = StrategyFactory.getByType(param.getCycle()); - catalogMetaDataFetchTaskSchedule.setCronExpression(api.funcDeal(catalogMetaDataFetchTaskSchedule.getParam())); + commonTaskSchedule.setCronExpression(api.funcDeal(commonTaskSchedule.getParam())); log.info("job schedule param: {}", paramStr); break; @@ -232,25 +232,26 @@ private void updateCatalogTaskScheduleParam(CatalogMetaDataFetchTaskSchedule cat if (!isValid) { throw new DataVinesServerException(Status.SCHEDULE_CRON_IS_INVALID_ERROR, param.getCrontab()); } - catalogMetaDataFetchTaskSchedule.setStatus(true); - catalogMetaDataFetchTaskSchedule.setParam(paramStr); - catalogMetaDataFetchTaskSchedule.setCronExpression(param.getCrontab()); + commonTaskSchedule.setStatus(true); + commonTaskSchedule.setParam(paramStr); + commonTaskSchedule.setCronExpression(param.getCrontab()); break; case OFFLINE: - catalogMetaDataFetchTaskSchedule.setStatus(false); + commonTaskSchedule.setStatus(false); break; default: throw new DataVinesServerException(Status.SCHEDULE_TYPE_NOT_VALIDATE_ERROR, type); } } - private ScheduleJobInfo getScheduleJobInfo(CatalogMetaDataFetchTaskSchedule catalogMetaDataFetchTaskSchedule) { + private ScheduleJobInfo getScheduleJobInfo(CommonTaskSchedule commonTaskSchedule) { return new ScheduleJobInfo( ScheduleJobType.CATALOG, - catalogMetaDataFetchTaskSchedule.getDataSourceId(), - catalogMetaDataFetchTaskSchedule.getId(), - catalogMetaDataFetchTaskSchedule.getCronExpression(), - catalogMetaDataFetchTaskSchedule.getStartTime(), - catalogMetaDataFetchTaskSchedule.getEndTime()); + commonTaskSchedule.getTaskType(), + commonTaskSchedule.getDataSourceId(), + commonTaskSchedule.getId(), + commonTaskSchedule.getCronExpression(), + commonTaskSchedule.getStartTime(), + commonTaskSchedule.getEndTime()); } } \ No newline at end of file diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskServiceImpl.java similarity index 51% rename from datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskServiceImpl.java rename to datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskServiceImpl.java index ec086383c..f6ce0bf16 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogMetaDataFetchTaskServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CommonTaskServiceImpl.java @@ -29,14 +29,15 @@ import io.datavines.core.exception.DataVinesServerException; import io.datavines.server.api.dto.bo.catalog.CatalogRefresh; import io.datavines.server.api.dto.vo.catalog.CatalogMetaDataFetchTaskVO; +import io.datavines.server.enums.CommonTaskType; import io.datavines.server.enums.FetchType; import io.datavines.server.registry.RegistryHolder; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; -import io.datavines.server.repository.mapper.CatalogMetaDataFetchTaskMapper; -import io.datavines.server.repository.service.CatalogMetaDataFetchCommandService; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskScheduleService; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskService; +import io.datavines.server.repository.entity.CommonTaskCommand; +import io.datavines.server.repository.entity.CommonTask; +import io.datavines.server.repository.mapper.CommonTaskMapper; +import io.datavines.server.repository.service.CommonTaskCommandService; +import io.datavines.server.repository.service.CommonTaskScheduleService; +import io.datavines.server.repository.service.CommonTaskService; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -49,16 +50,16 @@ import static io.datavines.common.ConfigConstants.DATABASE; import static io.datavines.common.ConfigConstants.TABLE; -@Service("catalogMetaDataFetchTaskService") -public class CatalogMetaDataFetchTaskServiceImpl - extends ServiceImpl - implements CatalogMetaDataFetchTaskService { +@Service("commonTaskService") +public class CommonTaskServiceImpl + extends ServiceImpl + implements CommonTaskService { @Autowired - private CatalogMetaDataFetchCommandService catalogMetaDataFetchCommandService; + private CommonTaskCommandService commonTaskCommandService; @Autowired - private CatalogMetaDataFetchTaskScheduleService catalogMetaDataFetchTaskScheduleService; + private CommonTaskScheduleService commonTaskScheduleService; @Autowired private RegistryHolder registryHolder; @@ -69,12 +70,13 @@ public long refreshCatalog(CatalogRefresh catalogRefresh) { Long taskId = 0L; registryHolder.blockUtilAcquireLock("1028"); - QueryWrapper queryWrapper = new QueryWrapper<>(); + QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(CatalogMetaDataFetchTask::getStatus,0) - .eq(CatalogMetaDataFetchTask::getDataSourceId, catalogRefresh.getDatasourceId()) - .eq(CatalogMetaDataFetchTask::getParameter, JSONUtils.toJsonString(catalogRefresh)); - List oldTaskList = baseMapper.selectList(queryWrapper); + queryWrapper.lambda().eq(CommonTask::getStatus,0) + .eq(CommonTask::getTaskType, CommonTaskType.CATALOG_METADATA_FETCH) + .eq(CommonTask::getDataSourceId, catalogRefresh.getDatasourceId()) + .eq(CommonTask::getParameter, JSONUtils.toJsonString(catalogRefresh)); + List oldTaskList = baseMapper.selectList(queryWrapper); if (CollectionUtils.isNotEmpty(oldTaskList)) { registryHolder.release("1028"); @@ -82,14 +84,15 @@ public long refreshCatalog(CatalogRefresh catalogRefresh) { } //生成任务之前需要检查是否有相同的任务在执行 LocalDateTime now = LocalDateTime.now(); - CatalogMetaDataFetchTask catalogMetaDataFetchTask = new CatalogMetaDataFetchTask(); - catalogMetaDataFetchTask.setParameter(JSONUtils.toJsonString(catalogRefresh)); - catalogMetaDataFetchTask.setDataSourceId(catalogRefresh.getDatasourceId()); - catalogMetaDataFetchTask.setStatus(0); - catalogMetaDataFetchTask.setExecuteHost(NetUtils.getAddr( + CommonTask commonTask = new CommonTask(); + commonTask.setTaskType(catalogRefresh.getTaskType()); + commonTask.setParameter(JSONUtils.toJsonString(catalogRefresh)); + commonTask.setDataSourceId(catalogRefresh.getDatasourceId()); + commonTask.setStatus(0); + commonTask.setExecuteHost(NetUtils.getAddr( CommonPropertyUtils.getInt(CommonPropertyUtils.SERVER_PORT, CommonPropertyUtils.SERVER_PORT_DEFAULT))); - - String parameter = catalogMetaDataFetchTask.getParameter(); + commonTask.setTaskType(catalogRefresh.getTaskType()); + String parameter = commonTask.getParameter(); if (StringUtils.isNotEmpty(parameter)) { Map parameterMap = JSONUtils.toMap(parameter); if (parameterMap != null) { @@ -97,7 +100,7 @@ public long refreshCatalog(CatalogRefresh catalogRefresh) { String table = parameterMap.get(TABLE); if (StringUtils.isEmpty(database) && StringUtils.isEmpty(table)) { - catalogMetaDataFetchTask.setType(FetchType.DATASOURCE); + commonTask.setType(FetchType.DATASOURCE); } if (StringUtils.isEmpty(database) && StringUtils.isNotEmpty(table)) { @@ -105,42 +108,42 @@ public long refreshCatalog(CatalogRefresh catalogRefresh) { } if (StringUtils.isNotEmpty(database) && StringUtils.isEmpty(table)) { - catalogMetaDataFetchTask.setDatabaseName(database); - catalogMetaDataFetchTask.setType(FetchType.DATABASE); + commonTask.setDatabaseName(database); + commonTask.setType(FetchType.DATABASE); } if (StringUtils.isNotEmpty(database) && StringUtils.isNotEmpty(table)) { - catalogMetaDataFetchTask.setTableName(table); - catalogMetaDataFetchTask.setDatabaseName(database); - catalogMetaDataFetchTask.setType(FetchType.TABLE); + commonTask.setTableName(table); + commonTask.setDatabaseName(database); + commonTask.setType(FetchType.TABLE); } } } - catalogMetaDataFetchTask.setSubmitTime(now); - catalogMetaDataFetchTask.setCreateTime(now); - catalogMetaDataFetchTask.setUpdateTime(now); + commonTask.setSubmitTime(now); + commonTask.setCreateTime(now); + commonTask.setUpdateTime(now); - baseMapper.insert(catalogMetaDataFetchTask); + baseMapper.insert(commonTask); - CatalogMetaDataFetchCommand catalogMetaDataFetchCommand = new CatalogMetaDataFetchCommand(); - catalogMetaDataFetchCommand.setTaskId(catalogMetaDataFetchTask.getId()); - catalogMetaDataFetchCommand.setCreateTime(now); - catalogMetaDataFetchCommand.setUpdateTime(now); - catalogMetaDataFetchCommandService.create(catalogMetaDataFetchCommand); - taskId = catalogMetaDataFetchTask.getId(); + CommonTaskCommand commonTaskCommand = new CommonTaskCommand(); + commonTaskCommand.setTaskId(commonTask.getId()); + commonTaskCommand.setCreateTime(now); + commonTaskCommand.setUpdateTime(now); + commonTaskCommandService.create(commonTaskCommand); + taskId = commonTask.getId(); registryHolder.release("1028"); return taskId; } @Override - public int update(CatalogMetaDataFetchTask catalogMetaDataFetchTask) { - return baseMapper.updateById(catalogMetaDataFetchTask); + public int update(CommonTask commonTask) { + return baseMapper.updateById(commonTask); } @Override - public CatalogMetaDataFetchTask getById(long id) { + public CommonTask getById(long id) { return baseMapper.selectById(id); } @@ -150,17 +153,17 @@ public Long killCatalogTask(Long catalogTaskId) { } @Override - public List listNeedFailover(String host) { - return baseMapper.selectList(new QueryWrapper().lambda() - .eq(CatalogMetaDataFetchTask::getExecuteHost, host) - .in(CatalogMetaDataFetchTask::getStatus, ExecutionStatus.RUNNING_EXECUTION.getCode(), ExecutionStatus.SUBMITTED_SUCCESS.getCode())); + public List listNeedFailover(String host) { + return baseMapper.selectList(new QueryWrapper().lambda() + .eq(CommonTask::getExecuteHost, host) + .in(CommonTask::getStatus, ExecutionStatus.RUNNING_EXECUTION.getCode(), ExecutionStatus.SUBMITTED_SUCCESS.getCode())); } @Override - public List listTaskNotInServerList(List hostList) { - return baseMapper.selectList(new QueryWrapper().lambda() - .notIn(CatalogMetaDataFetchTask::getExecuteHost, hostList) - .in(CatalogMetaDataFetchTask::getStatus,ExecutionStatus.RUNNING_EXECUTION.getCode(), ExecutionStatus.SUBMITTED_SUCCESS.getCode())); + public List listTaskNotInServerList(List hostList) { + return baseMapper.selectList(new QueryWrapper().lambda() + .notIn(CommonTask::getExecuteHost, hostList) + .in(CommonTask::getStatus,ExecutionStatus.RUNNING_EXECUTION.getCode(), ExecutionStatus.SUBMITTED_SUCCESS.getCode())); } @Override @@ -171,21 +174,21 @@ public String getTaskExecuteHost(Long catalogTaskId) { @Override @Transactional(rollbackFor = Exception.class) public boolean deleteByDataSourceId(long dataSourceId) { - remove(new QueryWrapper().lambda().eq(CatalogMetaDataFetchTask::getDataSourceId, dataSourceId)); - catalogMetaDataFetchTaskScheduleService.deleteByDataSourceId(dataSourceId); + remove(new QueryWrapper().lambda().eq(CommonTask::getDataSourceId, dataSourceId)); + commonTaskScheduleService.deleteByDataSourceId(dataSourceId); return false; } @Override public LocalDateTime getRefreshTime(long dataSourceId, String databaseName, String tableName) { - CatalogMetaDataFetchTask task = null; - QueryWrapper queryWrapper = new QueryWrapper<>(); + CommonTask task = null; + QueryWrapper queryWrapper = new QueryWrapper<>(); LocalDateTime refreshTime = null; - queryWrapper.lambda().eq(CatalogMetaDataFetchTask::getDataSourceId,dataSourceId) - .eq(CatalogMetaDataFetchTask::getType,FetchType.DATASOURCE) - .orderByDesc(CatalogMetaDataFetchTask::getCreateTime).last("limit 1"); + queryWrapper.lambda().eq(CommonTask::getDataSourceId,dataSourceId) + .eq(CommonTask::getType,FetchType.DATASOURCE) + .orderByDesc(CommonTask::getCreateTime).last("limit 1"); task = getOne(queryWrapper); if (task != null) { refreshTime = task.getCreateTime(); @@ -193,10 +196,10 @@ public LocalDateTime getRefreshTime(long dataSourceId, String databaseName, Stri if (StringUtils.isNotEmpty(databaseName)) { queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(CatalogMetaDataFetchTask::getDataSourceId,dataSourceId) - .eq(CatalogMetaDataFetchTask::getDatabaseName, databaseName) - .eq(CatalogMetaDataFetchTask::getType,FetchType.DATABASE) - .orderByDesc(CatalogMetaDataFetchTask::getCreateTime).last("limit 1"); + queryWrapper.lambda().eq(CommonTask::getDataSourceId,dataSourceId) + .eq(CommonTask::getDatabaseName, databaseName) + .eq(CommonTask::getType,FetchType.DATABASE) + .orderByDesc(CommonTask::getCreateTime).last("limit 1"); task = getOne(queryWrapper); if (task != null) { if (refreshTime == null || task.getCreateTime().isAfter(refreshTime)) { @@ -207,10 +210,10 @@ public LocalDateTime getRefreshTime(long dataSourceId, String databaseName, Stri if (StringUtils.isNotEmpty(databaseName) && StringUtils.isNotEmpty(tableName)) { queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(CatalogMetaDataFetchTask::getDataSourceId,dataSourceId) - .eq(CatalogMetaDataFetchTask::getDatabaseName, databaseName) - .eq(CatalogMetaDataFetchTask::getTableName,tableName) - .orderByDesc(CatalogMetaDataFetchTask::getCreateTime).last("limit 1"); + queryWrapper.lambda().eq(CommonTask::getDataSourceId,dataSourceId) + .eq(CommonTask::getDatabaseName, databaseName) + .eq(CommonTask::getTableName,tableName) + .orderByDesc(CommonTask::getCreateTime).last("limit 1"); task = getOne(queryWrapper); if (task != null) { if (refreshTime == null || task.getCreateTime().isAfter(refreshTime)) { @@ -223,8 +226,8 @@ public LocalDateTime getRefreshTime(long dataSourceId, String databaseName, Stri } @Override - public IPage getFetchTaskPage(Long datasourceId,Integer pageNumber, Integer pageSize) { + public IPage getFetchTaskPage(Long datasourceId, String taskType, Integer pageNumber, Integer pageSize) { Page page = new Page<>(pageNumber, pageSize); - return baseMapper.getJobExecutionPage(page, datasourceId); + return baseMapper.getJobExecutionPage(page, datasourceId, taskType); } } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ConfigServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ConfigServiceImpl.java index e5be50c91..64fd4c30f 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ConfigServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ConfigServiceImpl.java @@ -72,7 +72,8 @@ public int update(ConfigUpdate configUpdate) throws DataVinesServerException { throw new DataVinesServerException(Status.CONFIG_NOT_EXIST_ERROR, configUpdate.getVarKey()); } - BeanUtils.copyProperties(configUpdate, config); + config.setVarKey(configUpdate.getVarKey()); + config.setVarValue(configUpdate.getVarValue()); config.setUpdateBy(ContextHolder.getUserId()); config.setUpdateTime(LocalDateTime.now()); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java index 0f683ee18..b24089d6b 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java @@ -32,6 +32,7 @@ import io.datavines.server.api.dto.bo.datasource.DataSourceCreate; import io.datavines.server.api.dto.bo.datasource.DataSourceUpdate; import io.datavines.server.api.dto.vo.DataSourceVO; +import io.datavines.server.enums.CommonTaskType; import io.datavines.server.repository.entity.DataSource; import io.datavines.server.repository.mapper.DataSourceMapper; import io.datavines.server.repository.service.*; @@ -64,7 +65,7 @@ public class DataSourceServiceImpl extends ServiceImpl implements JobExecutionResultReportRelService { + + @Override + public List listExecutionResultByReportId(Long reportId) { + return baseMapper.listExecutionResultByReportId(reportId); + } +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java index b3058a7b2..cc8adf59f 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java @@ -16,6 +16,7 @@ */ package io.datavines.server.repository.service.impl; +import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -317,7 +318,7 @@ public List getMetricExecutionDashBoard(Long jobId, St ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(result.getResultFormula()); MetricExecutionDashBoard executionDashBoard = new MetricExecutionDashBoard(); - executionDashBoard.setValue(resultFormula.getResult(result.getActualValue(), Objects.isNull(result.getExpectedValue()) ? 0 : result.getExpectedValue())); + executionDashBoard.setValue(resultFormula.getResult(result.getActualValue(), Objects.isNull(result.getExpectedValue()) ? BigDecimal.valueOf(0) : result.getExpectedValue())); executionDashBoard.setType(resultFormula.getType().getDescription()); executionDashBoard.setDatetime(result.getCreateTime().toString()); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExternalService.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExternalService.java index 5f48a13b8..4fc43b973 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExternalService.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExternalService.java @@ -28,8 +28,8 @@ import io.datavines.server.repository.entity.Job; import io.datavines.server.repository.entity.JobExecution; import io.datavines.server.repository.entity.JobExecutionResult; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; +import io.datavines.server.repository.entity.CommonTaskCommand; +import io.datavines.server.repository.entity.CommonTask; import io.datavines.server.repository.service.*; import io.datavines.server.utils.DefaultDataSourceInfoUtils; import lombok.extern.slf4j.Slf4j; @@ -62,10 +62,10 @@ public class JobExternalService { private ActualValuesService actualValuesService; @Autowired - private CatalogMetaDataFetchCommandService catalogMetaDataFetchCommandService; + private CommonTaskCommandService commonTaskCommandService; @Autowired - private CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService; + private CommonTaskService commonTaskService; @Autowired private DataSourceService dataSourceService; @@ -85,8 +85,8 @@ public Command getCommand(int totalSlot, int currentSlot){ return commandService.getOne(totalSlot, currentSlot); } - public CatalogMetaDataFetchCommand getCatalogCommand(int totalSlot, int currentSlot){ - return catalogMetaDataFetchCommandService.getOne(totalSlot, currentSlot); + public CommonTaskCommand getCatalogCommand(int totalSlot, int currentSlot){ + return commonTaskCommandService.getOne(totalSlot, currentSlot); } public int deleteCommandById(long id){ @@ -94,15 +94,15 @@ public int deleteCommandById(long id){ } public int deleteCatalogCommandById(long id){ - return catalogMetaDataFetchCommandService.deleteById(id); + return commonTaskCommandService.deleteById(id); } public JobExecution executeCommand(Command command){ return jobExecutionService.getById(command.getJobExecutionId()); } - public CatalogMetaDataFetchTask executeCatalogCommand(CatalogMetaDataFetchCommand command){ - return catalogMetaDataFetchTaskService.getById(command.getTaskId()); + public CommonTask executeCatalogCommand(CommonTaskCommand command){ + return commonTaskService.getById(command.getTaskId()); } public int updateJobExecution(JobExecution jobExecution){ @@ -208,8 +208,8 @@ public DataSourceService getDataSourceService() { return dataSourceService; } - public CatalogMetaDataFetchTaskService getCatalogTaskService() { - return catalogMetaDataFetchTaskService; + public CommonTaskService getCatalogTaskService() { + return commonTaskService; } public JobExecutionResultService getJobExecutionResultService() { diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java new file mode 100644 index 000000000..45be4740e --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.repository.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import io.datavines.common.entity.job.BaseJobParameter; +import io.datavines.common.enums.OperatorType; +import io.datavines.common.exception.DataVinesException; +import io.datavines.common.utils.DateUtils; +import io.datavines.common.utils.JSONUtils; +import io.datavines.common.utils.ParameterUtils; +import io.datavines.common.utils.StringUtils; +import io.datavines.core.utils.LanguageUtils; +import io.datavines.metric.api.*; +import io.datavines.server.api.dto.bo.job.JobQualityReportDashboardParam; +import io.datavines.server.api.dto.vo.*; +import io.datavines.server.enums.DataQualityLevel; +import io.datavines.server.enums.DqJobExecutionState; +import io.datavines.server.repository.entity.*; +import io.datavines.server.repository.mapper.JobExecutionResultMapper; +import io.datavines.server.repository.mapper.JobQualityReportMapper; +import io.datavines.server.repository.service.JobExecutionService; +import io.datavines.server.repository.service.JobQualityReportService; +import io.datavines.server.repository.service.JobExecutionResultReportRelService; +import io.datavines.spi.PluginLoader; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +import static io.datavines.common.ConfigConstants.*; + +@Service("jobQualityReportService") +public class JobQualityReportServiceImpl extends ServiceImpl implements JobQualityReportService { + + @Autowired + private JobExecutionResultMapper jobExecutionResultMapper; + + @Autowired + private JobExecutionResultReportRelService jobExecutionResultReportRelService; + + @Autowired + private JobQualityReportMapper jobQualityReportMapper; + + @Autowired + private JobExecutionService jobExecutionService; + + @Override + public boolean generateQualityReport(Long datasourceId) { + String yesterday = DateUtils.format(DateUtils.addDays(DateUtils.getCurrentDate(),-1),DateUtils.YYYY_MM_DD); + List jobExecutionResultList = jobExecutionResultMapper.listByDatasourceIdAndTimeRange(datasourceId,yesterday + " 00:00:00", yesterday + " 23:59:59"); + if (CollectionUtils.isEmpty(jobExecutionResultList)) { + return true; + } + Map> key2MetricMap = new HashMap<>(); + Map> key2IdMap = new HashMap<>(); + Map key2ScoreMap = new HashMap<>(); + for (JobExecutionResult executionResult : jobExecutionResultList) { + String key = null; + if (StringUtils.isEmpty(executionResult.getColumnName())) { + key = String.format("%s@#@%s",executionResult.getDatabaseName(), executionResult.getTableName()); + } else { + key = String.format("%s@#@%s@#@%s",executionResult.getDatabaseName(), executionResult.getTableName(), executionResult.getColumnName()); + } + + Set metricSet = key2MetricMap.get(key); + if (CollectionUtils.isEmpty(metricSet)) { + metricSet = new HashSet<>(); + } + + if (metricSet.contains(executionResult.getMetricName())) { + continue; + } + + List resultIds = key2IdMap.get(key); + if (CollectionUtils.isEmpty(resultIds)) { + resultIds = new ArrayList<>(); + } + + resultIds.add(executionResult.getId()); + key2IdMap.put(key, resultIds); + + BigDecimal scoreSum = key2ScoreMap.get(key); + if (scoreSum == null) { + scoreSum = new BigDecimal(0); + } + + scoreSum = scoreSum.add(executionResult.getScore()); + key2ScoreMap.put(key, scoreSum); + + metricSet.add(executionResult.getMetricName()); + key2MetricMap.put(key,metricSet); + } + + if (MapUtils.isNotEmpty(key2ScoreMap)) { + for (Map.Entry entry : key2ScoreMap.entrySet()) { + String key = entry.getKey(); + BigDecimal score = entry.getValue(); + + List executionResultIds = key2IdMap.get(key); + + String databaseName = "--"; + String tableName = "--"; + String columnName = "--"; + String[] keyValues = key.split("@#@"); + if (keyValues.length == 2) { + databaseName = keyValues[0]; + tableName = keyValues[1]; + } else if (keyValues.length == 3) { + databaseName = keyValues[0]; + tableName = keyValues[1]; + columnName = keyValues[2]; + } else { + continue; + } + + JobQualityReport jobQualityReport = null; + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(JobQualityReport::getDatasourceId, datasourceId); + queryWrapper.eq(JobQualityReport::getDatabaseName, databaseName); + queryWrapper.eq(JobQualityReport::getTableName, tableName); + queryWrapper.eq(JobQualityReport::getColumnName, columnName); + queryWrapper.eq(JobQualityReport::getReportDate, yesterday); + jobQualityReport = jobQualityReportMapper.selectOne(queryWrapper); + + if (jobQualityReport == null) { + jobQualityReport = new JobQualityReport(); + jobQualityReport.setReportDate(LocalDate.parse(yesterday)); + jobQualityReport.setScore(score.divide(new BigDecimal(executionResultIds.size()),4, RoundingMode.UP)); + jobQualityReport.setDatabaseName(databaseName); + jobQualityReport.setTableName(tableName); + jobQualityReport.setColumnName(columnName); + jobQualityReport.setDatasourceId(datasourceId); + jobQualityReport.setCreateTime(LocalDateTime.now()); + jobQualityReport.setUpdateTime(LocalDateTime.now()); + jobQualityReport.setEntityLevel(MetricLevel.COLUMN.getDescription()); + jobQualityReportMapper.insert(jobQualityReport); + } else { + jobQualityReport.setScore(score.divide(new BigDecimal(executionResultIds.size()),4, RoundingMode.UP)); + jobQualityReport.setUpdateTime(LocalDateTime.now()); + jobQualityReportMapper.updateById(jobQualityReport); + } + + Long qualityReportId = jobQualityReport.getId(); + + List relList = new ArrayList<>(); + for (Long executionResultId : executionResultIds) { + JobExecutionResultReportRel rel = new JobExecutionResultReportRel(); + rel.setQualityReportId(qualityReportId); + rel.setJobExecutionResultId(executionResultId); + relList.add(rel); + } + + if (CollectionUtils.isNotEmpty(relList)) { + jobExecutionResultReportRelService.remove(new QueryWrapper().eq("quality_report_id",qualityReportId)); + jobExecutionResultReportRelService.saveBatch(relList); + } + } + } + + // 删除datasource_id 下所有表级别的评分 + jobQualityReportMapper.delete(new QueryWrapper().eq("entity_level","table").eq("datasource_id",datasourceId)); + // 根据datasource_id,database_name,table_name 进行group by,计算得到表的分数 + List tableScoreList = jobQualityReportMapper.listTableScoreGroupByDatasource(datasourceId, yesterday); + if (CollectionUtils.isNotEmpty(tableScoreList)) { + saveBatch(tableScoreList); + } + + jobQualityReportMapper.delete(new QueryWrapper().eq("entity_level","database").eq("datasource_id",datasourceId)); + // 根据datasource_id,database_name 进行 group by,计算得到数据库的分数 + List databaseScoreList = jobQualityReportMapper.listDbScoreGroupByDatasource(datasourceId, yesterday); + if (CollectionUtils.isNotEmpty(databaseScoreList)) { + saveBatch(databaseScoreList); + } + + jobQualityReportMapper.delete(new QueryWrapper().eq("entity_level","datasource").eq("datasource_id", datasourceId)); + // 根据datasource_id,database_name 进行 group by,计算得到数据库的分数 + List datasourceScoreList = jobQualityReportMapper.listDatasourceScoreGroupByDatasource(datasourceId, yesterday); + if (CollectionUtils.isNotEmpty(datasourceScoreList)) { + saveBatch(datasourceScoreList); + } + + return true; + } + + @Override + public JobQualityReportScore getScoreByCondition(JobQualityReportDashboardParam dashboardParam) { + + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + if (dashboardParam == null) { + throw new DataVinesException("param can not be null"); + } + + queryWrapper.eq(dashboardParam.getDatasourceId()!= null, JobQualityReport::getDatasourceId, dashboardParam.getDatasourceId()); + + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getSchemaName()), JobQualityReport::getDatabaseName, dashboardParam.getSchemaName()); + if (StringUtils.isEmpty(dashboardParam.getTableName())) { + queryWrapper.eq(JobQualityReport::getTableName, "--"); + } else { + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getTableName()), JobQualityReport::getTableName, dashboardParam.getTableName()); + } + + if (StringUtils.isEmpty(dashboardParam.getReportDate())) { + String yesterday = DateUtils.format(DateUtils.addDays(DateUtils.getCurrentDate(),-1),DateUtils.YYYY_MM_DD); + queryWrapper.eq(JobQualityReport::getReportDate, yesterday); + } else { + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getReportDate()), JobQualityReport::getReportDate, dashboardParam.getReportDate()); + } + + List jobQualityReports = jobQualityReportMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(jobQualityReports)) { + return null; + } + JobQualityReportScore reportScore = new JobQualityReportScore(); + reportScore.setScore(new BigDecimal(0)); + reportScore.setQualityLevel(DataQualityLevel.UNQUALIFIED.getZhDescription()); + + JobQualityReport report = jobQualityReports.get(0); + reportScore.setScore(report.getScore()); + reportScore.setQualityLevel(DataQualityLevel.getQualityLevelByScore(report.getScore()).getZhDescription()); + return reportScore; + } + + @Override + public JobQualityReportScoreTrend getScoreTrendByCondition(JobQualityReportDashboardParam dashboardParam) { + if (dashboardParam == null) { + throw new DataVinesException("param can not be null"); + } + + JobQualityReportScoreTrend scoreTrend = new JobQualityReportScoreTrend(); + String startDateStr = ""; + String endDateStr = ""; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + if (StringUtils.isEmpty(dashboardParam.getReportDate())) { + startDateStr = DateUtils.format(DateUtils.addDays(new Date(), -7),"yyyy-MM-dd"); + endDateStr = DateUtils.format(DateUtils.addDays(new Date(), -1),"yyyy-MM-dd"); + } else { + endDateStr = dashboardParam.getReportDate(); + LocalDate endDate = LocalDate.parse(endDateStr, formatter); + ZonedDateTime zonedDateTime = endDate.atStartOfDay(ZoneId.systemDefault()); + Date date = Date.from(zonedDateTime.toInstant()); + startDateStr = DateUtils.format(DateUtils.addDays(date,-6),"yyyy-MM-dd"); + } + + LocalDate startDate = LocalDate.parse(startDateStr, formatter); + LocalDate endDate = LocalDate.parse(endDateStr, formatter); + + List dateList = new ArrayList<>(); + LocalDate currentDate = startDate; + + while (!currentDate.isAfter(endDate)) { + dateList.add(currentDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + currentDate = currentDate.plusDays(1); + } + + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(dashboardParam.getDatasourceId()!= null, JobQualityReport::getDatasourceId, dashboardParam.getDatasourceId()); + + if (StringUtils.isEmpty(dashboardParam.getTableName())) { + queryWrapper.eq(JobQualityReport::getDatabaseName, "--"); + } else { + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getSchemaName()), JobQualityReport::getDatabaseName, dashboardParam.getSchemaName()); + } + + if (StringUtils.isEmpty(dashboardParam.getTableName())) { + queryWrapper.eq(JobQualityReport::getTableName, "--"); + } else { + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getTableName()), JobQualityReport::getTableName, dashboardParam.getTableName()); + } + + queryWrapper.between(JobQualityReport::getReportDate, startDateStr, endDateStr); + queryWrapper.orderByAsc(JobQualityReport::getReportDate); + List reportList = list(queryWrapper); + + Map date2Score = new HashMap<>(); + if (CollectionUtils.isNotEmpty(reportList)) { + reportList.forEach(it -> { + date2Score.put(it.getReportDate().toString(), it.getScore()); + }); + } else { + return scoreTrend; + } + + List scoreList = new ArrayList<>(); + + dateList.forEach(date -> { + BigDecimal score = date2Score.get(date); + if (score == null) { + scoreList.add(BigDecimal.valueOf(0)); + } else { + scoreList.add(score); + } + }); + + scoreTrend.setDateList(dateList); + scoreTrend.setScoreList(scoreList); + + return scoreTrend; + } + + @Override + public IPage getQualityReportPage(JobQualityReportDashboardParam dashboardParam) { + Page page = new Page<>(dashboardParam.getPageNumber(), dashboardParam.getPageSize()); + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(JobQualityReport::getDatasourceId, dashboardParam.getDatasourceId()); + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getSchemaName()),JobQualityReport::getDatabaseName,dashboardParam.getSchemaName()); + queryWrapper.eq(StringUtils.isNotEmpty(dashboardParam.getTableName()),JobQualityReport::getTableName,dashboardParam.getTableName()); + if (StringUtils.isEmpty(dashboardParam.getReportDate())) { + String yesterday = DateUtils.format(DateUtils.addDays(DateUtils.getCurrentDate(),-1),DateUtils.YYYY_MM_DD); + queryWrapper.eq(JobQualityReport::getReportDate, yesterday); + } else { + queryWrapper.eq(JobQualityReport::getReportDate, dashboardParam.getReportDate()); + } + + if (StringUtils.isNotEmpty(dashboardParam.getTableName())) { + queryWrapper.eq(JobQualityReport::getEntityLevel, "column"); + } else { + queryWrapper.eq(JobQualityReport::getEntityLevel, "table"); + } + + return page(page, queryWrapper).convert(jobQualityReport -> { + JobQualityReportVO jobQualityReportVO = new JobQualityReportVO(); + BeanUtils.copyProperties(jobQualityReport, jobQualityReportVO); + return jobQualityReportVO; + }); + } + + @Override + public List listColumnExecution(Long reportId) { + List executionResults = jobExecutionResultReportRelService.listExecutionResultByReportId(reportId); + if (CollectionUtils.isEmpty(executionResults)) { + return Collections.emptyList(); + } + + return executionResults.stream().map(jobExecutionResult -> { + Map parameters = new HashMap<>(); + parameters.put(ACTUAL_VALUE, String.valueOf(jobExecutionResult.getActualValue())); + parameters.put(EXPECTED_VALUE, String.valueOf(jobExecutionResult.getExpectedValue())); + parameters.put(THRESHOLD, String.valueOf(jobExecutionResult.getThreshold())); + parameters.put(OPERATOR,OperatorType.of(jobExecutionResult.getOperator()).getSymbol()); + JobExecution jobExecution = jobExecutionService.getById(jobExecutionResult.getJobExecutionId()); + JobExecutionResultVO jobExecutionResultVO = new JobExecutionResultVO(); + ResultFormula resultFormula = + PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(jobExecutionResult.getResultFormula()); + String resultFormulaFormat = resultFormula.getResultFormat(!LanguageUtils.isZhContext())+" ${operator} ${threshold}"; + + jobExecutionResultVO.setCheckSubject(jobExecutionResult.getDatabaseName() + "." + jobExecutionResult.getTableName() + "." + jobExecutionResult.getColumnName()); + jobExecutionResultVO.setCheckResult(DqJobExecutionState.of(jobExecutionResult.getState()).getDescription(!LanguageUtils.isZhContext())); + SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobExecutionResult.getMetricName()); + if (!"multi_table_value_comparison".equalsIgnoreCase(sqlMetric.getName())) { + ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin(jobExecution.getEngineType() + "_" + jobExecutionResult.getExpectedType()); + jobExecutionResultVO.setExpectedType(expectedValue.getNameByLanguage(!LanguageUtils.isZhContext())); + } + jobExecutionResultVO.setMetricName(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext())); + + jobExecutionResultVO.setResultFormulaFormat(ParameterUtils.convertParameterPlaceholders(resultFormulaFormat, parameters)); + jobExecutionResultVO.setScore(jobExecutionResult.getScore()); + jobExecutionResultVO.setExecutionTime(jobExecutionResult.getCreateTime()); + return jobExecutionResultVO; + }).collect(Collectors.toList()); + } + +} diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobScheduleServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobScheduleServiceImpl.java index d65fe6f4f..ad0b2f934 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobScheduleServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobScheduleServiceImpl.java @@ -169,6 +169,7 @@ private JobSchedule update(JobScheduleCreateOrUpdate jobScheduleUpdate) throws D private ScheduleJobInfo getScheduleJobInfo(JobSchedule jobSchedule, Job job) { return new ScheduleJobInfo( ScheduleJobType.DATA_QUALITY, + null, job.getDataSourceId(), job.getId(), jobSchedule.getCronExpression(), diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskContext.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskContext.java similarity index 68% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskContext.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskContext.java index f2a3cc78f..0eaa67b66 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskContext.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskContext.java @@ -14,19 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler; +import io.datavines.server.enums.CommonTaskType; import lombok.Data; @Data -public class CatalogTaskContext { +public class CommonTaskContext { - private CatalogMetaDataFetchRequest catalogMetaDataFetchRequest; + private CommonTaskType commonTaskType; + + private CommonTaskRequest commonTaskRequest; private Long catalogTaskId; - public CatalogTaskContext(CatalogMetaDataFetchRequest catalogMetaDataFetchRequest, Long catalogTaskId) { - this.catalogMetaDataFetchRequest = catalogMetaDataFetchRequest; + public CommonTaskContext(CommonTaskType commonTaskType, CommonTaskRequest commonTaskRequest, Long catalogTaskId) { + this.commonTaskType = commonTaskType; + this.commonTaskRequest = commonTaskRequest; this.catalogTaskId = catalogTaskId; } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskFailover.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskFailover.java similarity index 61% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskFailover.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskFailover.java index b56bea6a6..da795a034 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskFailover.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskFailover.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata; +package io.datavines.server.scheduler; import io.datavines.common.utils.CommonPropertyUtils; import io.datavines.common.utils.NetUtils; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskService; +import io.datavines.server.repository.entity.CommonTask; +import io.datavines.server.repository.service.CommonTaskService; import io.datavines.server.utils.SpringApplicationContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -27,31 +27,31 @@ import java.util.List; @Slf4j -public class CatalogMetaDataFetchTaskFailover { +public class CommonTaskFailover { - private final CatalogMetaDataFetchTaskService metaDataFetchTaskService; + private final CommonTaskService commonTaskService; - private final CatalogMetaDataFetchTaskManager metaDataFetchTaskManager; + private final CommonTaskManager commonTaskManager; - public CatalogMetaDataFetchTaskFailover(CatalogMetaDataFetchTaskManager metaDataFetchTaskManager) { - this.metaDataFetchTaskService = SpringApplicationContext.getBean(CatalogMetaDataFetchTaskService.class); - this.metaDataFetchTaskManager = metaDataFetchTaskManager; + public CommonTaskFailover(CommonTaskManager commonTaskManager) { + this.commonTaskService = SpringApplicationContext.getBean(CommonTaskService.class); + this.commonTaskManager = commonTaskManager; } public void handleMetaDataFetchTaskFailover(String host) { - List needFailoverTaskList = metaDataFetchTaskService.listNeedFailover(host); + List needFailoverTaskList = commonTaskService.listNeedFailover(host); innerHandleMetaDataFetchTaskFailover(needFailoverTaskList); } - private void innerHandleMetaDataFetchTaskFailover(List needFailover) { + private void innerHandleMetaDataFetchTaskFailover(List needFailover) { if (CollectionUtils.isNotEmpty(needFailover)) { needFailover.forEach(task -> { task.setExecuteHost(NetUtils.getAddr( CommonPropertyUtils.getInt(CommonPropertyUtils.SERVER_PORT, CommonPropertyUtils.SERVER_PORT_DEFAULT))); - metaDataFetchTaskService.updateById(task); + commonTaskService.updateById(task); try { - metaDataFetchTaskManager.putCatalogTask(task); + commonTaskManager.putCommonTask(task); } catch (Exception e) { log.error("put the task need failover into manager error : ", e); } @@ -60,7 +60,7 @@ private void innerHandleMetaDataFetchTaskFailover(List } public void handleMetaDataFetchTaskFailover(List hostList) { - List needFailoverTaskList = metaDataFetchTaskService.listTaskNotInServerList(hostList); + List needFailoverTaskList = commonTaskService.listTaskNotInServerList(hostList); innerHandleMetaDataFetchTaskFailover(needFailoverTaskList); } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskManager.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskManager.java similarity index 52% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskManager.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskManager.java index 87177406a..3af9d8b56 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskManager.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskManager.java @@ -14,18 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata; +package io.datavines.server.scheduler; import io.datavines.common.utils.*; import io.datavines.server.enums.FetchType; -import io.datavines.server.catalog.metadata.task.CatalogTaskContext; -import io.datavines.server.catalog.metadata.task.CatalogTaskResponse; -import io.datavines.server.catalog.metadata.task.CatalogTaskResponseQueue; -import io.datavines.server.catalog.metadata.task.CatalogMetaDataFetchRequest; +import io.datavines.server.scheduler.metadata.CatalogMetaDataFetchTaskRunner; import io.datavines.server.repository.entity.DataSource; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; -import io.datavines.server.repository.service.CatalogMetaDataFetchTaskService; +import io.datavines.server.repository.entity.CommonTask; +import io.datavines.server.repository.service.CommonTaskService; import io.datavines.server.repository.service.impl.JobExternalService; +import io.datavines.server.scheduler.report.DataQualityReportTaskRunner; import io.datavines.server.utils.NamedThreadFactory; import io.datavines.server.utils.SpringApplicationContext; import lombok.extern.slf4j.Slf4j; @@ -37,25 +35,25 @@ import java.util.concurrent.LinkedBlockingQueue; @Slf4j -public class CatalogMetaDataFetchTaskManager { +public class CommonTaskManager { - private final LinkedBlockingQueue taskQueue = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue taskQueue = new LinkedBlockingQueue<>(); - private final CatalogTaskResponseQueue responseQueue = - SpringApplicationContext.getBean(CatalogTaskResponseQueue.class); + private final CommonTaskResponseQueue responseQueue = + SpringApplicationContext.getBean(CommonTaskResponseQueue.class); - private final CatalogMetaDataFetchTaskService catalogMetaDataFetchTaskService = - SpringApplicationContext.getBean(CatalogMetaDataFetchTaskService.class); + private final CommonTaskService commonTaskService = + SpringApplicationContext.getBean(CommonTaskService.class); private final JobExternalService jobExternalService = SpringApplicationContext.getBean(JobExternalService.class); private final ExecutorService taskExecuteService; - public CatalogMetaDataFetchTaskManager() { + public CommonTaskManager() { this.taskExecuteService = Executors.newFixedThreadPool( CommonPropertyUtils.getInt(CommonPropertyUtils.METADATA_FETCH_EXEC_THREADS,CommonPropertyUtils.METADATA_FETCH_EXEC_THREADS_DEFAULT), - new NamedThreadFactory("CatalogMetaDataFetchExecutor-Execute-Thread")); + new NamedThreadFactory("CommonTaskExecutor-Execute-Thread")); } public void start() { @@ -70,12 +68,22 @@ class TaskExecutor extends Thread { public void run() { while(Stopper.isRunning()) { try { - CatalogTaskContext catalogTaskContext = taskQueue.take(); - taskExecuteService.execute(new CatalogMetaDataFetchTaskRunner(catalogTaskContext)); - CatalogMetaDataFetchTask catalogMetaDataFetchTask = catalogMetaDataFetchTaskService.getById(catalogTaskContext.getCatalogTaskId()); - if (catalogMetaDataFetchTask != null) { - catalogMetaDataFetchTask.setStartTime(LocalDateTime.now()); - catalogMetaDataFetchTaskService.update(catalogMetaDataFetchTask); + CommonTaskContext commonTaskContext = taskQueue.take(); + switch (commonTaskContext.getCommonTaskType()) { + case CATALOG_METADATA_FETCH: + taskExecuteService.execute(new CatalogMetaDataFetchTaskRunner(commonTaskContext)); + break; + case DATA_QUALITY_REPORT: + taskExecuteService.execute(new DataQualityReportTaskRunner(commonTaskContext)); + break; + default: + break; + } + + CommonTask commonTask = commonTaskService.getById(commonTaskContext.getCatalogTaskId()); + if (commonTask != null) { + commonTask.setStartTime(LocalDateTime.now()); + commonTaskService.update(commonTask); } ThreadUtils.sleep(1000); } catch(Exception e) { @@ -95,13 +103,13 @@ class TaskResponseOperator extends Thread { public void run() { while (Stopper.isRunning()) { try { - CatalogTaskResponse taskResponse = responseQueue.take(); + CommonTaskResponse taskResponse = responseQueue.take(); log.info("CatalogTaskResponse: " + JSONUtils.toJsonString(taskResponse)); - CatalogMetaDataFetchTask catalogMetaDataFetchTask = catalogMetaDataFetchTaskService.getById(taskResponse.getCatalogTaskId()); - if (catalogMetaDataFetchTask != null) { - catalogMetaDataFetchTask.setStatus(taskResponse.getStatus()); - catalogMetaDataFetchTask.setEndTime(LocalDateTime.now()); - catalogMetaDataFetchTaskService.update(catalogMetaDataFetchTask); + CommonTask commonTask = commonTaskService.getById(taskResponse.getCatalogTaskId()); + if (commonTask != null) { + commonTask.setStatus(taskResponse.getStatus()); + commonTask.setEndTime(LocalDateTime.now()); + commonTaskService.update(commonTask); } ThreadUtils.sleep(1000); } catch(Exception e) { @@ -111,40 +119,40 @@ public void run() { } } - public void putCatalogTask(CatalogMetaDataFetchTask catalogMetaDataFetchTask) throws InterruptedException { - if (catalogMetaDataFetchTask == null) { + public void putCommonTask(CommonTask commonTask) throws InterruptedException { + if (commonTask == null) { return; } - Long dataSourceId = catalogMetaDataFetchTask.getDataSourceId(); + Long dataSourceId = commonTask.getDataSourceId(); DataSource dataSource = jobExternalService.getDataSourceService().getDataSourceById(dataSourceId); if (dataSource == null) { return; } - CatalogMetaDataFetchRequest catalogMetaDataFetchRequest = new CatalogMetaDataFetchRequest(); - catalogMetaDataFetchRequest.setDataSource(dataSource); - catalogMetaDataFetchRequest.setFetchType(FetchType.DATASOURCE); + CommonTaskRequest commonTaskRequest = new CommonTaskRequest(); + commonTaskRequest.setDataSource(dataSource); + commonTaskRequest.setFetchType(FetchType.DATASOURCE); - String parameter = catalogMetaDataFetchTask.getParameter(); + String parameter = commonTask.getParameter(); if (StringUtils.isNotEmpty(parameter)) { Map parameterMap = JSONUtils.toMap(parameter); if (parameterMap != null) { String database = parameterMap.get("database"); if (StringUtils.isNotEmpty(database)) { - catalogMetaDataFetchRequest.setDatabase(database); - catalogMetaDataFetchRequest.setFetchType(FetchType.DATABASE); + commonTaskRequest.setDatabase(database); + commonTaskRequest.setFetchType(FetchType.DATABASE); } String table = parameterMap.get("table"); if (StringUtils.isNotEmpty(table)) { - catalogMetaDataFetchRequest.setTable(table); - catalogMetaDataFetchRequest.setFetchType(FetchType.TABLE); + commonTaskRequest.setTable(table); + commonTaskRequest.setFetchType(FetchType.TABLE); } } } - CatalogTaskContext catalogTaskContext = new CatalogTaskContext(catalogMetaDataFetchRequest, catalogMetaDataFetchTask.getId()); - taskQueue.put(catalogTaskContext); + CommonTaskContext commonTaskContext = new CommonTaskContext(commonTask.getTaskType(), commonTaskRequest, commonTask.getId()); + taskQueue.put(commonTaskContext); } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchRequest.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskRequest.java similarity index 91% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchRequest.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskRequest.java index ef882375d..7867d196b 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchRequest.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskRequest.java @@ -14,14 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler; import io.datavines.server.enums.FetchType; import io.datavines.server.repository.entity.DataSource; import lombok.Data; @Data -public class CatalogMetaDataFetchRequest { +public class CommonTaskRequest { private DataSource dataSource; diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponse.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponse.java similarity index 86% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponse.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponse.java index 37b77a000..0af66ecde 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponse.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponse.java @@ -14,18 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler; import lombok.Data; @Data -public class CatalogTaskResponse { +public class CommonTaskResponse { private Long catalogTaskId; private int status; - public CatalogTaskResponse(Long catalogTaskId, int status) { + public CommonTaskResponse(Long catalogTaskId, int status) { this.catalogTaskId = catalogTaskId; this.status = status; } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponseQueue.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponseQueue.java similarity index 71% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponseQueue.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponseQueue.java index 1599f8144..abef57664 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogTaskResponseQueue.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskResponseQueue.java @@ -14,22 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler; import org.springframework.stereotype.Component; import java.util.concurrent.LinkedBlockingQueue; @Component -public class CatalogTaskResponseQueue { +public class CommonTaskResponseQueue { - private final LinkedBlockingQueue responseQueue = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue responseQueue = new LinkedBlockingQueue<>(); - public boolean add(CatalogTaskResponse catalogTaskResponse) { - return responseQueue.add(catalogTaskResponse); + public boolean add(CommonTaskResponse commonTaskResponse) { + return responseQueue.add(commonTaskResponse); } - public CatalogTaskResponse take() throws InterruptedException { + public CommonTaskResponse take() throws InterruptedException { return responseQueue.take(); } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskScheduler.java b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskScheduler.java similarity index 78% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskScheduler.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskScheduler.java index 87881ef1c..6c140411b 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskScheduler.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/CommonTaskScheduler.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata; +package io.datavines.server.scheduler; import io.datavines.common.utils.*; import io.datavines.server.registry.Register; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchCommand; -import io.datavines.server.repository.entity.catalog.CatalogMetaDataFetchTask; +import io.datavines.server.repository.entity.CommonTaskCommand; +import io.datavines.server.repository.entity.CommonTask; import io.datavines.server.repository.service.impl.JobExternalService; import io.datavines.server.utils.SpringApplicationContext; import lombok.extern.slf4j.Slf4j; @@ -28,29 +28,29 @@ import static io.datavines.common.utils.CommonPropertyUtils.*; @Slf4j -public class CatalogMetaDataFetchTaskScheduler extends Thread { +public class CommonTaskScheduler extends Thread { private static final int[] RETRY_BACKOFF = {1, 2, 3, 5, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}; private final JobExternalService jobExternalService; - private final CatalogMetaDataFetchTaskManager catalogMetaDataFetchTaskManager; + private final CommonTaskManager commonTaskManager; private final Register register; - public CatalogMetaDataFetchTaskScheduler(CatalogMetaDataFetchTaskManager catalogMetaDataFetchTaskManager, Register register){ + public CommonTaskScheduler(CommonTaskManager commonTaskManager, Register register){ this.jobExternalService = SpringApplicationContext.getBean(JobExternalService.class); - this.catalogMetaDataFetchTaskManager = catalogMetaDataFetchTaskManager; + this.commonTaskManager = commonTaskManager; this.register = register; } @Override public void run() { - log.info("catalog metadata fetch task scheduler started"); + log.info("common task scheduler started"); int retryNum = 0; while (Stopper.isRunning()) { - CatalogMetaDataFetchCommand command = null; + CommonTaskCommand command = null; try { boolean runCheckFlag = OSUtils.checkResource( CommonPropertyUtils.getDouble(MAX_CPU_LOAD_AVG, MAX_CPU_LOAD_AVG_DEFAULT), @@ -64,10 +64,10 @@ public void run() { command = jobExternalService.getCatalogCommand(register.getTotalSlot(), register.getSlot()); if (command != null) { - CatalogMetaDataFetchTask task = jobExternalService.executeCatalogCommand(command); + CommonTask task = jobExternalService.executeCatalogCommand(command); if (task != null) { log.info("start submit catalog metadata fetch task : {} ", JSONUtils.toJsonString(task)); - catalogMetaDataFetchTaskManager.putCatalogTask(task); + commonTaskManager.putCommonTask(task); log.info(String.format("submit success, catalog metadata fetch task : %s", task.getParameter()) ); } else { log.warn("catalog metadata fetch task {} is null", command.getTaskId()); diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskRunner.java b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/CatalogMetaDataFetchTaskRunner.java similarity index 59% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskRunner.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/metadata/CatalogMetaDataFetchTaskRunner.java index c88c3ea57..64b79f9d8 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/CatalogMetaDataFetchTaskRunner.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/CatalogMetaDataFetchTaskRunner.java @@ -14,38 +14,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata; +package io.datavines.server.scheduler.metadata; import io.datavines.common.enums.ExecutionStatus; -import io.datavines.server.catalog.metadata.task.CatalogMetaDataFetchExecutorImpl; -import io.datavines.server.catalog.metadata.task.CatalogTaskContext; -import io.datavines.server.catalog.metadata.task.CatalogTaskResponse; -import io.datavines.server.catalog.metadata.task.CatalogTaskResponseQueue; +import io.datavines.server.scheduler.metadata.task.CatalogMetaDataFetchExecutorImpl; +import io.datavines.server.scheduler.CommonTaskContext; +import io.datavines.server.scheduler.CommonTaskResponse; +import io.datavines.server.scheduler.CommonTaskResponseQueue; import io.datavines.server.utils.SpringApplicationContext; import lombok.extern.slf4j.Slf4j; @Slf4j public class CatalogMetaDataFetchTaskRunner implements Runnable { - private final CatalogTaskContext taskContext; + private final CommonTaskContext taskContext; - private final CatalogTaskResponseQueue responseQueue = - SpringApplicationContext.getBean(CatalogTaskResponseQueue.class); + private final CommonTaskResponseQueue responseQueue = + SpringApplicationContext.getBean(CommonTaskResponseQueue.class); - public CatalogMetaDataFetchTaskRunner(CatalogTaskContext taskContext) { + public CatalogMetaDataFetchTaskRunner(CommonTaskContext taskContext) { this.taskContext = taskContext; } @Override public void run() { - CatalogMetaDataFetchExecutorImpl fetchTask = new CatalogMetaDataFetchExecutorImpl(taskContext.getCatalogMetaDataFetchRequest()); + CatalogMetaDataFetchExecutorImpl fetchTask = new CatalogMetaDataFetchExecutorImpl(taskContext.getCommonTaskRequest()); try { fetchTask.execute(); log.info("fetch metadata finished"); - responseQueue.add(new CatalogTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.SUCCESS.getCode())); + responseQueue.add(new CommonTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.SUCCESS.getCode())); } catch (Exception e) { log.error("fetch metadata error: ", e); - responseQueue.add(new CatalogTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.FAILURE.getCode())); + responseQueue.add(new CommonTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.FAILURE.getCode())); } } } diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutor.java b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutor.java similarity index 94% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutor.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutor.java index 465b53ca4..c4250e87b 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutor.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutor.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler.metadata.task; public interface CatalogMetaDataFetchExecutor { diff --git a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java similarity index 99% rename from datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java rename to datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java index 4ae0f36a2..5667ca292 100644 --- a/datavines-server/src/main/java/io/datavines/server/catalog/metadata/task/CatalogMetaDataFetchExecutorImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.server.catalog.metadata.task; +package io.datavines.server.scheduler.metadata.task; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import io.datavines.common.CommonConstants; @@ -40,6 +40,7 @@ import io.datavines.server.repository.service.CatalogEntityInstanceService; import io.datavines.server.repository.service.CatalogEntityRelService; import io.datavines.server.repository.service.CatalogSchemaChangeService; +import io.datavines.server.scheduler.CommonTaskRequest; import io.datavines.server.utils.SpringApplicationContext; import io.datavines.spi.PluginLoader; import lombok.extern.slf4j.Slf4j; @@ -57,7 +58,7 @@ @Slf4j public class CatalogMetaDataFetchExecutorImpl implements CatalogMetaDataFetchExecutor { - private final CatalogMetaDataFetchRequest request; + private final CommonTaskRequest request; private final ConnectorFactory connectorFactory; @@ -69,7 +70,7 @@ public class CatalogMetaDataFetchExecutorImpl implements CatalogMetaDataFetchExe private final DataSource dataSource; - public CatalogMetaDataFetchExecutorImpl(CatalogMetaDataFetchRequest request) { + public CatalogMetaDataFetchExecutorImpl(CommonTaskRequest request) { this.request = request; if (request.getDataSource() == null) { diff --git a/datavines-server/src/main/java/io/datavines/server/scheduler/report/DataQualityReportTaskRunner.java b/datavines-server/src/main/java/io/datavines/server/scheduler/report/DataQualityReportTaskRunner.java new file mode 100644 index 000000000..c74f10fa4 --- /dev/null +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/report/DataQualityReportTaskRunner.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.server.scheduler.report; + +import io.datavines.common.enums.ExecutionStatus; +import io.datavines.server.repository.service.JobQualityReportService; +import io.datavines.server.scheduler.CommonTaskContext; +import io.datavines.server.scheduler.CommonTaskResponse; +import io.datavines.server.scheduler.CommonTaskResponseQueue; +import io.datavines.server.utils.SpringApplicationContext; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DataQualityReportTaskRunner implements Runnable { + + private final CommonTaskContext taskContext; + + private final CommonTaskResponseQueue responseQueue = + SpringApplicationContext.getBean(CommonTaskResponseQueue.class); + + public DataQualityReportTaskRunner(CommonTaskContext taskContext) { + this.taskContext = taskContext; + } + + @Override + public void run() { + try { + JobQualityReportService jobQualityReportService = SpringApplicationContext.getBean(JobQualityReportService.class); + jobQualityReportService.generateQualityReport(taskContext.getCommonTaskRequest().getDataSource().getId()); + log.info("data quality report generate finished"); + responseQueue.add(new CommonTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.SUCCESS.getCode())); + } catch (Exception e) { + log.error("data quality report generate error: ", e); + responseQueue.add(new CommonTaskResponse(taskContext.getCatalogTaskId(), ExecutionStatus.FAILURE.getCode())); + } + } +} diff --git a/datavines-server/src/main/resources/mapper/CatalogMetaDataFetchTaskMapper.xml b/datavines-server/src/main/resources/mapper/CommonTaskMapper.xml similarity index 86% rename from datavines-server/src/main/resources/mapper/CatalogMetaDataFetchTaskMapper.xml rename to datavines-server/src/main/resources/mapper/CommonTaskMapper.xml index 330de758c..540259b39 100644 --- a/datavines-server/src/main/resources/mapper/CatalogMetaDataFetchTaskMapper.xml +++ b/datavines-server/src/main/resources/mapper/CommonTaskMapper.xml @@ -18,14 +18,17 @@ --> - + - select * from dv_catalog_metadata_fetch_task + select * from dv_common_task datasource_id = #{datasourceId} + + and task_type = #{taskType} + diff --git a/datavines-server/src/main/resources/mapper/JobExecutionResultMapper.xml b/datavines-server/src/main/resources/mapper/JobExecutionResultMapper.xml index 6088cbab5..6396ce236 100644 --- a/datavines-server/src/main/resources/mapper/JobExecutionResultMapper.xml +++ b/datavines-server/src/main/resources/mapper/JobExecutionResultMapper.xml @@ -30,5 +30,4 @@ - diff --git a/datavines-ui/Editor/components/Database/Detail/dashBoard.tsx b/datavines-ui/Editor/components/Database/Detail/dashBoard.tsx index 0926990e1..abbb57645 100644 --- a/datavines-ui/Editor/components/Database/Detail/dashBoard.tsx +++ b/datavines-ui/Editor/components/Database/Detail/dashBoard.tsx @@ -1,5 +1,5 @@ import * as echarts from 'echarts/core'; -import {LineChart, BarChart, PieChart} from 'echarts/charts'; +import {LineChart, BarChart, PieChart, GaugeChart} from 'echarts/charts'; import { ToolboxComponent, @@ -29,6 +29,7 @@ echarts.use([ CanvasRenderer, BarChart, PieChart, + GaugeChart, LegendComponent, ToolboxComponent ]); diff --git a/datavines-ui/Editor/components/Database/Detail/index.tsx b/datavines-ui/Editor/components/Database/Detail/index.tsx index b20975325..2acf2345c 100644 --- a/datavines-ui/Editor/components/Database/Detail/index.tsx +++ b/datavines-ui/Editor/components/Database/Detail/index.tsx @@ -69,13 +69,13 @@ const Index = (props: any, ref:any) => { const initialTableItems = [ { label: intl.formatMessage({ id: 'editor_dv_DashBoard' }), children: , key: '1' }, { - label: intl.formatMessage({ id: 'editor_dv_Configuation' }), + label: intl.formatMessage({ id: 'editor_dv_Configuration' }), children: { changeTabKey(value); setActiveKey(value); }} - style={{ marginBottom: '0px' }} + style={{ marginBottom: '0px',marginTop :'20px' }} className="dv-tab-list" /> diff --git a/datavines-ui/Editor/components/Database/Detail/runs.tsx b/datavines-ui/Editor/components/Database/Detail/runs.tsx index 9150b3e92..4a9920508 100644 --- a/datavines-ui/Editor/components/Database/Detail/runs.tsx +++ b/datavines-ui/Editor/components/Database/Detail/runs.tsx @@ -5,11 +5,12 @@ import { Table } from 'antd'; import { useIntl } from 'react-intl'; import { ColumnsType } from 'antd/es/table'; import { $http } from '@/http'; -import { IF } from '@/common'; -import { TJobsInstanceTableItem } from '@/type/JobsInstance'; +import {IF, useWatch} from '@/common'; +import {TJobsInstanceTableData, TJobsInstanceTableItem} from '@/type/JobsInstance'; import { useInstanceErrorDataModal } from '@/view/Main/HomeDetail/Jobs/useInstanceErrorDataModal'; import { useInstanceResult } from '@/view/Main/HomeDetail/Jobs/useInstanceResult'; import { useLogger } from '@/view/Main/HomeDetail/Jobs/useLogger'; +import {defaultRender} from "utils/helper"; // eslint-disable-next-line react/no-unused-prop-types const Index = (props: any, ref:any) => { @@ -17,6 +18,10 @@ const Index = (props: any, ref:any) => { useImperativeHandle(ref, () => ({ getData })); const intl = useIntl(); const [loading, setLoading] = useState(false); + const [pageParams, setPageParams] = useState({ + pageNumber: 1, + pageSize: 10, + }); const { Render: RenderErrorDataModal, show: showErrorDataModal } = useInstanceErrorDataModal({}); const { Render: RenderResultModal, show: showResultModal } = useInstanceResult({}); const { Render: RenderLoggerModal, show: showLoggerModal } = useLogger({}); @@ -39,46 +44,120 @@ const Index = (props: any, ref:any) => { const onErrorData = (record: TJobsInstanceTableItem) => { showErrorDataModal(record); }; - const [tableData, setTableData] = useState([]); - const columns: ColumnsType = [{ - title: 'Task Name', - dataIndex: 'name', - key: 'name', - }, { - title: 'Status', - dataIndex: 'status', - key: 'status', - }, { - title: 'Update Time', - dataIndex: 'updateTime', - key: 'updateTime', - }, { - title: intl.formatMessage({ id: 'common_action' }), - fixed: 'right', - key: 'right', - dataIndex: 'right', - width: 240, - render: (text: string, record: any) => ( - <> - - { onStop(record); }}>{intl.formatMessage({ id: 'jobs_task_stop_btn' })} - - { onLog(record); }}>{intl.formatMessage({ id: 'jobs_task_log_btn' })} - { onResult(record); }}>{intl.formatMessage({ id: 'jobs_task_result' })} - { onErrorData(record); }}>{intl.formatMessage({ id: 'jobs_task_error_data' })} - - ), - }]; - const getData = async () => { - setLoading(true); - const res = await $http.post('/job/execution/page', { - jobId: id, - pageNumber: 1, - pageSize: 999, + + const onChange = ({ current, pageSize }: any) => { + setPageParams({ + pageNumber: current, + pageSize, }); - setLoading(false); - setTableData(res.records); + getData(); }; + + const [tableData, setTableData] = useState({ list: [], total: 0 }); + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_name' }), + dataIndex: 'name', + key: 'name', + width: 250, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'schemaName', + key: 'schemaName', + width: 100, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 150, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_column_name' }), + dataIndex: 'columnName', + key: 'columnName', + width: 150, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_metric_type' }), + dataIndex: 'metricType', + key: 'metricType', + width: 180, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_status' }), + dataIndex: 'status', + key: 'status', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_check_status' }), + dataIndex: 'checkState', + key: 'checkState', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_start_time' }), + dataIndex: 'startTime', + key: 'startTime', + width: 180, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_end_time' }), + dataIndex: 'endTime', + key: 'endTime', + width: 180, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'common_action' }), + fixed: 'right', + key: 'right', + dataIndex: 'right', + width: 200, + render: (text: string, record: TJobsInstanceTableItem) => ( + <> + + { onStop(record); }}>{intl.formatMessage({ id: 'jobs_task_stop_btn' })} + + { onLog(record); }}>{intl.formatMessage({ id: 'jobs_task_log_btn' })} + { onResult(record); }}>{intl.formatMessage({ id: 'jobs_task_result' })} + { onErrorData(record); }}>{intl.formatMessage({ id: 'jobs_task_error_data' })} + + ), + }, + ]; + + const getData = async (values: any =null) => { + try { + setLoading(true); + const res = (await $http.post('/job/execution/page', { + jobId: id, + ...pageParams + })) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + + useWatch([pageParams], () => { + getData(); + }, { immediate: true }); + useEffect(() => { getData(); }, []); @@ -88,12 +167,22 @@ const Index = (props: any, ref:any) => { marginTop: '20px', }} > - + size="middle" + loading={loading} rowKey="id" - dataSource={tableData} columns={columns} - loading={loading} + dataSource={tableData.list || []} + onChange={onChange} + pagination={{ + size: 'small', + total: tableData.total, + showSizeChanger: true, + current: pageParams.pageNumber, + pageSize: pageParams.pageSize, + }} /> + diff --git a/datavines-ui/Editor/components/Database/index.tsx b/datavines-ui/Editor/components/Database/index.tsx index 633883f4f..2758f457d 100644 --- a/datavines-ui/Editor/components/Database/index.tsx +++ b/datavines-ui/Editor/components/Database/index.tsx @@ -654,6 +654,7 @@ const Index = ({ onShowModal, afterClose }:DIndexProps) => { database: selectDatabases[1]?.name, datasourceId: id, table: selectDatabases[2] ? selectDatabases[2]?.name : '', + taskType: 'CATALOG_METADATA_FETCH' }); message.success(intl.formatMessage({ id: 'common_success' })); }; @@ -867,7 +868,7 @@ const Index = ({ onShowModal, afterClose }:DIndexProps) => { // eslint-disable-next-line react/no-unstable-nested-components expandedRowRender: tableItems[+activeTableKey]?.name === 'Profile' // eslint-disable-next-line react/no-unstable-nested-components - ? (record:{ dataType: string;uuid:string, type:string}) => ( + ? (record:{ dataType: string; uuid:string, type:string}) => ( { }} width="100%" style={{ height: 'auto' }} - api="catalog/profile" + api="catalog/profile/schedule" jobId={selectDatabases[currentIndex]?.uuid} + taskType={'-1'} /> , + icon: , component: lazy(() => import(/* webpackChunkName: 'view-detail-dashboard' */ '@/view/Main/HomeDetail/Dashboard')), }, 'dv-detail-editor': { diff --git a/datavines-ui/src/type/JobExecutionLogs.ts b/datavines-ui/src/type/JobExecutionLogs.ts index 4f8b745f6..782f916be 100644 --- a/datavines-ui/src/type/JobExecutionLogs.ts +++ b/datavines-ui/src/type/JobExecutionLogs.ts @@ -2,7 +2,7 @@ export type TJobExecutionLosTableItem = { id: string | number, jobType: string, name: string, - status: 'submitted' | 'running' | 'failure' | 'success' | 'kill', + status: string, updateTime: string, } diff --git a/datavines-ui/src/type/JobQualityReport.ts b/datavines-ui/src/type/JobQualityReport.ts new file mode 100644 index 000000000..82095936e --- /dev/null +++ b/datavines-ui/src/type/JobQualityReport.ts @@ -0,0 +1,13 @@ +export type TJobQualityReportTableItem = { + id: string | number, + databaseName: string, + tableName: string, + columnName: string, + score: number, + reportDate: string, +} + +export type TJobQualityReportTableData = { + list: TJobQualityReportTableItem[], + total: number +}; diff --git a/datavines-ui/src/type/JobsInstance.ts b/datavines-ui/src/type/JobsInstance.ts index 217b69216..dd5cd5613 100644 --- a/datavines-ui/src/type/JobsInstance.ts +++ b/datavines-ui/src/type/JobsInstance.ts @@ -2,7 +2,7 @@ export type TJobsInstanceTableItem = { id: string | number, jobType: string, name: string, - status: 'submitted' | 'running' | 'failure' | 'success' | 'kill', + status: string, updateTime: string, } diff --git a/datavines-ui/src/view/Main/Home/List/MetaDataFetcher.tsx b/datavines-ui/src/view/Main/Home/List/MetaDataFetcher.tsx index da2132557..f575e7451 100644 --- a/datavines-ui/src/view/Main/Home/List/MetaDataFetcher.tsx +++ b/datavines-ui/src/view/Main/Home/List/MetaDataFetcher.tsx @@ -30,8 +30,9 @@ const Index: React.FC = ({datasourceId, onSavaEnd, refreshKey}) => { const getData = async (values?: any, $pageParams?: any) => { try { setLoading(true); - const res = (await $http.get('/catalog/metadata/task/page', { + const res = (await $http.get('/common-task/page', { datasourceId: datasourceId, + taskType: 'CATALOG_METADATA_FETCH', ...($pageParams || pageParams), })) || []; setTableData({ @@ -134,7 +135,7 @@ const Index: React.FC = ({datasourceId, onSavaEnd, refreshKey}) => { type="line" size="small" > - + { @@ -144,10 +145,11 @@ const Index: React.FC = ({datasourceId, onSavaEnd, refreshKey}) => { style={{ height: 'auto' }} jobId={datasourceId} isShowPush - api="catalog/metadata" + api="common-task" + taskType={'CATALOG_METADATA_FETCH'} /> - + size="middle" diff --git a/datavines-ui/src/view/Main/HomeDetail/Dashboard/executionDashboard/index.tsx b/datavines-ui/src/view/Main/HomeDetail/Dashboard/executionDashboard/index.tsx new file mode 100644 index 000000000..0092e4962 --- /dev/null +++ b/datavines-ui/src/view/Main/HomeDetail/Dashboard/executionDashboard/index.tsx @@ -0,0 +1,564 @@ +import React, {useEffect, useState} from 'react'; +import { + Button, + Cascader, + Col, + DatePicker, DatePickerProps, + Row, + Select, + Table +} from "antd"; +import DashBoard from "@Editor/components/Database/Detail/dashBoard"; +import {Title} from "@/component"; +import {useIntl} from "react-intl"; +import * as echarts from 'echarts'; +import {useRouteMatch} from "react-router-dom"; +import {$http} from "@/http"; +import {useMount} from "@Editor/common"; +import {ColumnsType} from "antd/lib/table"; +import {TJobsInstanceTableItem} from "@/type/JobsInstance"; +import {defaultRender} from "utils/helper"; +import {useInstanceErrorDataModal} from "view/Main/HomeDetail/Jobs/useInstanceErrorDataModal"; +import {useInstanceResult} from "view/Main/HomeDetail/Jobs/useInstanceResult"; +import {useLogger} from "view/Main/HomeDetail/Jobs/useLogger"; +import { useWatch } from '@/common'; + +type TJobs = { + datasourceId?: any, +} + +interface Option { + value?: string | number | null; + label: React.ReactNode; + children?: Option[]; + isLeaf?: boolean; +} + +const app: any = {}; +const posList = [ + 'left', + 'right', + 'top', + 'bottom', + 'inside', + 'insideTop', + 'insideLeft', + 'insideRight', + 'insideBottom', + 'insideTopLeft', + 'insideTopRight', + 'insideBottomLeft', + 'insideBottomRight' +] as const; + +app.configParameters = { + rotate: { + min: -90, + max: 90 + }, + align: { + options: { + left: 'left', + center: 'center', + right: 'right' + } + }, + verticalAlign: { + options: { + top: 'top', + middle: 'middle', + bottom: 'bottom' + } + }, + position: { + options: posList.reduce(function (map, pos) { + map[pos] = pos; + return map; + }, {} as Record) + }, + distance: { + min: 0, + max: 100 + } +}; + +const ExecutionDashboard = ({ datasourceId }: TJobs) => { + const [dqBarOption, setDqBarOption] = useState(); + const [dqPieOption, setDqPieOption] = useState(); + + const intl = useIntl(); + const match = useRouteMatch(); + + const [entityParam, setEntityParam] = useState({ + schemaName: null, + tableName: null, + columnName: null + }); + const [pageParam, setPageParam] = useState({ + pageNumber : 1, + pageSize : 5 + }); + const [metricType, setMetricType] = useState(); + const [startTime, setStartTime] = useState(); + const [endTime, setEndTime] = useState(); + + const onEntitySelectChange = async (value: (string | number | null)[], selectedOptions: Option[]) => { + + if (value) { + if (value.length == 1) { + setEntityParam({ + schemaName:value[0] + }) + + } else if (value.length == 2) { + setEntityParam({ + schemaName : value[0], + tableName : value[1] + }) + } else if (value.length == 3) { + setEntityParam({ + schemaName : value[0], + tableName : value[1], + columnName : value[2] + }) + } + } else { + setEntityParam({ + schemaName : null, + tableName : null, + columnName : null + }) + } + }; + + const loadData = (selectedOptions: Option[]) => { + console.log(selectedOptions); + setTimeout(async () => { + const targetOption = selectedOptions[selectedOptions.length - 1]; + if (selectedOptions.length == 1) { + const tables = await $http.get(`/datasource/${(match.params as any).id}/${selectedOptions[0].value}/tables`); + let $reTables = tables ? JSON.parse(JSON.stringify(tables)) : []; + const $reTables1: ((prevState: never[]) => never[]) | { value: any; label: any; isLeaf:any;}[] = []; + $reTables.forEach((item: { name: any; }) => { + $reTables1.push({value: item.name, label: item.name,isLeaf:false}) + }) + targetOption.children = $reTables1; + setDataBases([...databases]) + } else if (selectedOptions.length == 2) { + + const columns = await $http.get(`/datasource/${(match.params as any).id}/${selectedOptions[0].value}/${selectedOptions[1].value}/columns`); + let $reColumns = columns ? JSON.parse(JSON.stringify(columns)) : []; + const $reColumns1: ((prevState: never[]) => never[]) | { value: any; label: any; isLeaf:any;}[] = []; + $reColumns.forEach((item: { name: any; }) => { + $reColumns1.push({value: item.name, label: item.name, isLeaf:true}) + }) + targetOption.children = $reColumns1; + setDataBases([...databases]) + } + },1000); + }; + + const [loading, setLoading] = useState(false); + + const [tableData, setTableData] = useState<{ list: TJobsInstanceTableItem[], total: number}>({ list: [], total: 0 }); + + const { Render: RenderErrorDataModal, show: showErrorDataModal } = useInstanceErrorDataModal({}); + const { Render: RenderResultModal, show: showResultModal } = useInstanceResult({}); + const { Render: RenderLoggerModal, show: showLoggerModal } = useLogger({}); + + const [databases, setDataBases] = useState([]); + const [metricList, setMetricList] = useState([]); + + const getJobExecutionData = async (pageParam1 :any) => { + try { + setLoading(true); + const res = (await $http.post('/job/execution/page', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + columnName : entityParam.columnName, + metricType : metricType, + datasourceId : datasourceId || (match.params as any).id, + pageNumber : pageParam1.pageNumber, + pageSize : pageParam1.pageSize, + status: 6, + startTime : startTime, + endTime : endTime + }, + )) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + + const getJobExecutionAggPie = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/execution/agg-pie', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + columnName : entityParam.columnName, + metricType : metricType, + datasourceId : datasourceId || (match.params as any).id, + startTime : startTime, + endTime : endTime + }, + )) || []; + console.log("agg pie res : ", res) + + const pieOption = { + tooltip: { + trigger: 'item' + }, + legend: { + top: '5%', + left: 'center' + }, + color: ['#ef6567', '#91cd77'], + series: [ + { + type: 'pie', + radius: ['40%', '70%'], + avoidLabelOverlap: false, + itemStyle: { + borderRadius: 10, + borderColor: '#fff', + borderWidth: 2 + }, + label: { + show: false, + position: 'center' + }, + emphasis: { + label: { + show: true, + fontSize: 40, + fontWeight: 'bold' + } + }, + labelLine: { + show: false + }, + data: res + } + ] + }; + setDqPieOption(pieOption) + + } catch (error) { + } finally { + setLoading(false); + } + }; + + const getJobExecutionTrendBar = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/execution/trend-bar', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + columnName : entityParam.columnName, + metricType : metricType, + datasourceId : datasourceId || (match.params as any).id, + startTime : startTime, + endTime : endTime + }, + )) || []; + console.log("agg bar res : ", res) + + const barOption = { + tooltip: { + trigger: 'axis' + }, + legend: { + data: ['All', 'Success', 'Failure'] + }, + grid: { + left: '3%', + right: '4%', + bottom: '3%', + containLabel: true + }, + toolbox: { + feature: { + saveAsImage: {} + } + }, + xAxis: { + type: 'category', + boundaryGap: false, + data: res.dateList + }, + yAxis: { + type: 'value' + }, + series: [ + { + name: 'All', + type: 'line', + data: res.allList + }, + { + name: 'Success', + type: 'line', + data: res.successList + }, + { + name: 'Failure', + type: 'line', + color: 'red', + data: res.failureList + } + ] + }; + setDqBarOption(barOption) + + } catch (error) { + } finally { + setLoading(false); + } + }; + + useEffect(() => { + + }, []); + + useWatch([(match.params as any).id], () =>{ + refreshData() + }) + + useMount(async () => { + refreshData() + }); + + const refreshData = async () => { + setEntityParam({ + schemaName: null, + tableName: null, + columnName: null + }) + + setMetricType(null) + setStartTime(null) + setEndTime(null) + + const $metricList = await $http.get('metric/list/DATA_QUALITY'); + let $reMetricList = $metricList ? JSON.parse(JSON.stringify($metricList)) : []; + const $reMetricList1: ((prevState: never[]) => never[]) | { value: any; label: any; }[] = []; + $reMetricList.forEach((item: { key: any; label: any; }) => { + return $reMetricList1?.push({value: item.key, label: item.label}); + }) + // @ts-ignore + setMetricList($reMetricList1); + console.log("metricList: ", $reMetricList) + const $datasourceId = datasourceId || (match.params as any).id + const $databases = await $http.get(`/datasource/${$datasourceId}/databases`); + let $reDatabases = $databases ? JSON.parse(JSON.stringify($databases)) : []; + const $reDatabases1: ((prevState: never[]) => never[]) | { value: any; label: any; isLeaf: any; }[] = []; + $reDatabases.forEach((item: { name: any; }) => { + return $reDatabases1.push({value: item.name, label: item.name, isLeaf: false}); + }) + // @ts-ignore + setDataBases($reDatabases1); + getJobExecutionData(pageParam); + getJobExecutionAggPie(); + getJobExecutionTrendBar(); + } + + const onPageChange = ({ current, pageSize }: any) => { + setPageParam({ + pageNumber : current, + pageSize : pageSize + }) + + getJobExecutionData({ + pageNumber : current, + pageSize : pageSize + }); + }; + + const onLog = (record: TJobsInstanceTableItem) => { + showLoggerModal(record); + }; + + const onResult = (record: TJobsInstanceTableItem) => { + showResultModal(record); + }; + + const onErrorData = (record: TJobsInstanceTableItem) => { + showErrorDataModal(record); + }; + + const onMetricSelectChange = (value: string) => { + setMetricType(value) + }; + + const onStartTimeChange : DatePickerProps['onChange'] = (date, dateString) => { + setStartTime(dateString) + }; + + const onEndTimeChange : DatePickerProps['onChange'] = (date, dateString) => { + setEndTime(dateString) + }; + + const onQueryClick = () => { + getJobExecutionData(pageParam); + getJobExecutionAggPie(); + getJobExecutionTrendBar(); + } + + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_name' }), + dataIndex: 'name', + key: 'name', + width: 300, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'schemaName', + key: 'schemaName', + width: 100, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_column_name' }), + dataIndex: 'columnName', + key: 'columnName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_metric_type' }), + dataIndex: 'metricType', + key: 'metricType', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_type' }), + dataIndex: 'jobType', + key: 'jobType', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_status' }), + dataIndex: 'status', + key: 'status', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_check_status' }), + dataIndex: 'checkState', + key: 'checkState', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_start_time' }), + dataIndex: 'startTime', + key: 'startTime', + width: 180, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_end_time' }), + dataIndex: 'endTime', + key: 'endTime', + width: 180, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'common_action' }), + fixed: 'right', + key: 'right', + dataIndex: 'right', + width: 200, + render: (text: string, record: TJobsInstanceTableItem) => ( + <> + { onLog(record); }}>{intl.formatMessage({ id: 'jobs_task_log_btn' })} + { onResult(record); }}>{intl.formatMessage({ id: 'jobs_task_result' })} + { onErrorData(record); }}>{intl.formatMessage({ id: 'jobs_task_error_data' })} + + ), + }, + ]; + return ( +
+
+ +
+ + - 开始时间 - 结束时间 - - - - -
- -
- - {intl.formatMessage({ id: 'quality_dashboard_profile' })} - - - - - - {intl.formatMessage({ id: 'quality_dashboard_trend' })} - - - - - - - - {intl.formatMessage({ id: 'quality_dashboard_failure_execution' })} - - - size="middle" - loading={loading} - rowKey="id" - columns={columns} - dataSource={tableData.list || []} - onChange={onPageChange} - pagination={{ - size: 'small', - total: tableData.total, - showSizeChanger: true, - current: pageParam.pageNumber, - pageSize: pageParam.pageSize, - pageSizeOptions: [5, 10, 20, 50, 100], - }} - /> - - - - - - +
+ { + setActiveKey(value); + }} + style={{ marginBottom: '0px',marginTop :'5px' }} + className="dv-tab-list" + > + + + + + + + + +
); }; diff --git a/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/ReportScheduler.tsx b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/ReportScheduler.tsx new file mode 100644 index 000000000..530130afa --- /dev/null +++ b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/ReportScheduler.tsx @@ -0,0 +1,175 @@ +import React, {useEffect, useState} from 'react'; +import { + Table +} from 'antd'; +import { useIntl } from 'react-intl'; +import Schedule from '@/view/Main/HomeDetail/Jobs/components/Schedule'; +import {Tabs} from "antd/lib"; +import TabPane from "antd/lib/tabs/TabPane"; +import {TJobsInstanceTableData, TJobsInstanceTableItem} from "@/type/JobsInstance"; +import {$http} from "@/http"; +import {IF, useMount} from "@Editor/common"; +import {ColumnsType} from "antd/lib/table"; +import {defaultRender} from "utils/helper"; + +type IndexProps = { + datasourceId: number | string, + onSavaEnd: any, + refreshKey: number, +} + +const Index: React.FC = ({datasourceId, onSavaEnd, refreshKey}) => { + const intl = useIntl(); + const [loading, setLoading] = useState(false); + const [tableData, setTableData] = useState({ list: [], total: 0 }); + const [pageParams, setPageParams] = useState({ + pageNumber: 1, + pageSize: 10, + }); + + const getData = async (values?: any, $pageParams?: any) => { + try { + setLoading(true); + const res = (await $http.get('/common-task/page', { + datasourceId: datasourceId, + taskType:'DATA_QUALITY_REPORT', + ...($pageParams || pageParams), + })) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + console.log("get execution page : {}", error) + } finally { + setLoading(false); + } + }; + useMount(() => { + getData(); + }); + useEffect(() =>{ + getData(); + }, [refreshKey]) + + const onChange = ({ current, pageSize }: any) => { + setPageParams({ + pageNumber: current, + pageSize, + }); + getData(null, { + pageNumber: current, + pageSize, + }); + }; + + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_type' }), + dataIndex: 'type', + key: 'type', + width: 100, + render: (text) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'databaseName', + key: 'databaseName', + width: 100, + render: (text) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 200, + render: (text) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_status' }), + dataIndex: 'status', + key: 'status', + width: 100, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_execute_host' }), + dataIndex: 'executeHost', + key: 'executeHost', + width: 140, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_schedule_time' }), + dataIndex: 'scheduleTime', + key: 'scheduleTime', + width: 200, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_submit_time' }), + dataIndex: 'submitTime', + key: 'submitTime', + width: 200, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_start_time' }), + dataIndex: 'startTime', + key: 'startTime', + width: 200, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_end_time' }), + dataIndex: 'endTime', + key: 'endTime', + width: 200, + render: (text: string) =>
{text || '--'}
, + } + ]; + + return ( + <> + + + + { + onSavaEnd(); + }} + width="100%" + style={{ height: 'auto' }} + jobId={datasourceId} + isShowPush + api="common-task" + taskType={'DATA_QUALITY_REPORT'} + /> + + + + + size="middle" + loading={loading} + rowKey="id" + columns={columns} + dataSource={tableData.list || []} + onChange={onChange} + pagination={{ + size: 'small', + total: tableData.total, + showSizeChanger: true, + current: pageParams.pageNumber, + pageSize: pageParams.pageSize, + }} + /> + + + + ) +}; + +export default Index; \ No newline at end of file diff --git a/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/index.tsx b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/index.tsx new file mode 100644 index 000000000..1e92b557e --- /dev/null +++ b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/index.tsx @@ -0,0 +1,529 @@ +import React, {useEffect, useState} from 'react'; +import { + Button, + Cascader, + Checkbox, + Col, + DatePicker, DatePickerProps, + Dropdown, + Form, + Input, + MenuProps, + message, Modal, + Row, + Select, + Table +} from "antd"; +import DashBoard from "@Editor/components/Database/Detail/dashBoard"; +import {Title} from "@/component"; +import {useIntl} from "react-intl"; +import * as echarts from 'echarts'; +import {useRouteMatch} from "react-router-dom"; +import {$http} from "@/http"; +import {useMount} from "@Editor/common"; +import {ColumnsType} from "antd/lib/table"; + +import {defaultRender} from "utils/helper"; +import {qualityReportDetailModal} from "view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal"; +import { useWatch } from '@/common'; +import {TJobQualityReportTableItem} from "@/type/JobQualityReport"; +import MetaDataFetcher from "view/Main/Home/List/MetaDataFetcher"; +import ReportScheduler from "view/Main/HomeDetail/Dashboard/qualityReportDashboard/ReportScheduler"; + +type TJobs = { + datasourceId?: any, +} + +interface Option { + value?: string | number | null; + label: React.ReactNode; + children?: Option[]; + isLeaf?: boolean; +} + +const app: any = {}; +const posList = [ + 'left', + 'right', + 'top', + 'bottom', + 'inside', + 'insideTop', + 'insideLeft', + 'insideRight', + 'insideBottom', + 'insideTopLeft', + 'insideTopRight', + 'insideBottomLeft', + 'insideBottomRight' +] as const; + +app.configParameters = { + rotate: { + min: -90, + max: 90 + }, + align: { + options: { + left: 'left', + center: 'center', + right: 'right' + } + }, + verticalAlign: { + options: { + top: 'top', + middle: 'middle', + bottom: 'bottom' + } + }, + position: { + options: posList.reduce(function (map, pos) { + map[pos] = pos; + return map; + }, {} as Record) + }, + distance: { + min: 0, + max: 100 + } +}; + +const QualityReportDashboard = ({ datasourceId }: TJobs) => { + const [dqBarOption, setDqBarOption] = useState(); + const [dqGaugeOption, setDqGaugeOption] = useState(); + + const intl = useIntl(); + const match = useRouteMatch(); + + const [entityParam, setEntityParam] = useState({ + schemaName: null, + tableName: null + }); + + const [pageParam, setPageParam] = useState({ + pageNumber : 1, + pageSize : 5 + }); + + const [startTime, setStartTime] = useState(); + const [endTime, setEndTime] = useState(); + + const [reportDate, setReportDate] = useState(); + + const onEntitySelectChange = async (value: (string | number | null)[], selectedOptions: Option[]) => { + + if (value) { + if (value.length == 1) { + setEntityParam({ + schemaName:value[0] + }) + + } else if (value.length == 2) { + setEntityParam({ + schemaName : value[0], + tableName : value[1] + }) + } + } else { + setEntityParam({ + schemaName : null, + tableName : null + }) + } + }; + + const loadData = (selectedOptions: Option[]) => { + console.log(selectedOptions); + setTimeout(async () => { + const targetOption = selectedOptions[selectedOptions.length - 1]; + if (selectedOptions.length == 1) { + const tables = await $http.get(`/datasource/${(match.params as any).id}/${selectedOptions[0].value}/tables`); + let $reTables = tables ? JSON.parse(JSON.stringify(tables)) : []; + const $reTables1: ((prevState: never[]) => never[]) | { value: any; label: any; isLeaf:any;}[] = []; + $reTables.forEach((item: { name: any; }) => { + $reTables1.push({value: item.name, label: item.name,isLeaf:false}) + }) + targetOption.children = $reTables1; + setDataBases([...databases]) + } else if (selectedOptions.length == 2) { + targetOption.children = []; + setDataBases([...databases]) + } + },1000); + }; + + const [loading, setLoading] = useState(false); + + const [tableData, setTableData] = useState<{ list: TJobQualityReportTableItem[], total: number}>({ list: [], total: 0 }); + + const { Render: RenderQualityReportDetailModal, show: showQualityReportDetailModal } = qualityReportDetailModal({}); + + const [databases, setDataBases] = useState([]); + + const [isScheduleOpen, setisScheduleOpen] = useState(false); + + const [refresh, setRefresh] = useState(0); + + const getJobExecutionData = async (pageParam1 :any) => { + try { + setLoading(true); + const res = (await $http.post('/job/quality-report/page', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + datasourceId : datasourceId || (match.params as any).id, + reportDate: reportDate, + pageNumber : pageParam1.pageNumber, + pageSize : pageParam1.pageSize + }, + )) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + + const getJobExecutionAggPie = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/quality-report/score', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + datasourceId : datasourceId || (match.params as any).id, + reportDate : reportDate + }, + )) || []; + console.log("quality score res : ", res) + + const gaugeOption = { + series: [ + { + type: 'gauge', + startAngle: 180, + endAngle: 0, + center: ['50%', '75%'], + radius: '90%', + min: 0, + max: 1, + splitNumber: 8, + axisLine: { + lineStyle: { + width: 6, + color: [ + [0.2, '#fd0513'], + [0.4, '#e3622b'], + [0.6, '#FDDD60'], + [0.8, '#58D9F9'], + [1, '#7CFFB2'] + ] + } + }, + pointer: { + icon: 'path://M12.8,0.7l12,40.1H0.7L12.8,0.7z', + length: '12%', + width: 20, + offsetCenter: [0, '-60%'], + itemStyle: { + color: 'auto' + } + }, + axisTick: { + length: 12, + lineStyle: { + color: 'auto', + width: 2 + } + }, + splitLine: { + length: 20, + lineStyle: { + color: 'auto', + width: 5 + } + }, + axisLabel: { + color: '#464646', + fontSize: 20, + distance: -60, + rotate: 'tangential', + formatter: function (value: number) { + return ''; + } + }, + title: { + offsetCenter: [0, '-10%'], + fontSize: 20 + }, + detail: { + fontSize: 30, + offsetCenter: [0, '-35%'], + valueAnimation: true, + formatter: function (value: number) { + return Math.round(value * 100) + ''; + }, + color: 'inherit' + }, + data: [ + { + value: res.score / 100, + name: res.qualityLevel + } + ] + } + ] + }; + setDqGaugeOption(gaugeOption) + + } catch (error) { + } finally { + setLoading(false); + } + }; + + const getJobExecutionTrendBar = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/quality-report/score-trend', { + schemaName : entityParam.schemaName, + tableName : entityParam.tableName, + datasourceId : datasourceId || (match.params as any).id, + reportDate : reportDate + }, + )) || []; + console.log("score trend res : ", res) + + const barOption = { + tooltip: { + trigger: 'axis' + }, + legend: {}, + xAxis: { + type: 'category', + boundaryGap: false, + data: res.dateList + }, + yAxis: { + type: 'value', + axisLabel: { + formatter: '{value}' + } + }, + series: [ + { + name: 'Score', + type: 'line', + data: res.scoreList, + markPoint: { + data: [ + { type: 'max', name: 'Max' }, + { type: 'min', name: 'Min' } + ] + }, + markLine: { + data: [{ type: 'average', name: 'Avg' }] + } + } + ] + }; + setDqBarOption(barOption) + + } catch (error) { + } finally { + setLoading(false); + } + }; + + useEffect(() => { + if(isScheduleOpen) { + setRefresh(prev => prev + 1); + } + }, [isScheduleOpen]) + + useWatch([(match.params as any).id], () =>{ + refreshData() + }) + + useMount(async () => { + refreshData() + }); + + const refreshData = async () => { + setEntityParam({ + schemaName: null, + tableName: null + }) + + setStartTime(null) + setEndTime(null) + + const $datasourceId = datasourceId || (match.params as any).id + const $databases = await $http.get(`/datasource/${$datasourceId}/databases`); + let $reDatabases = $databases ? JSON.parse(JSON.stringify($databases)) : []; + const $reDatabases1: ((prevState: never[]) => never[]) | { value: any; label: any; isLeaf: any; }[] = []; + $reDatabases.forEach((item: { name: any; }) => { + return $reDatabases1.push({value: item.name, label: item.name, isLeaf: false}); + }) + // @ts-ignore + setDataBases($reDatabases1); + getJobExecutionData(pageParam); + getJobExecutionAggPie(); + getJobExecutionTrendBar(); + } + + const onPageChange = ({ current, pageSize }: any) => { + setPageParam({ + pageNumber : current, + pageSize : pageSize + }) + + getJobExecutionData({ + pageNumber : current, + pageSize : pageSize + }); + }; + + const onShow = (record: TJobQualityReportTableItem) => { + showQualityReportDetailModal(record); + }; + + const onStartTimeChange : DatePickerProps['onChange'] = (date, dateString) => { + setReportDate(dateString) + }; + + const onEndTimeChange : DatePickerProps['onChange'] = (date, dateString) => { + setEndTime(dateString) + }; + + const onQueryClick = () => { + getJobExecutionData(pageParam); + getJobExecutionAggPie(); + getJobExecutionTrendBar(); + } + + const [uuid, setUuid] = useState(''); + const showSchedule = () => { + setisScheduleOpen(true); + setUuid(datasourceId || (match.params as any).id); + } + + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'databaseName', + key: 'databaseName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_score' }), + dataIndex: 'score', + key: 'score', + width: 200, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_report_date' }), + dataIndex: 'reportDate', + key: 'reportDate', + width: 200, + render: (text: string) =>
{text || '--'}
, + }, + { + title: intl.formatMessage({ id: 'common_action' }), + fixed: 'right', + key: 'right', + dataIndex: 'right', + width: 200, + render: (text: string, record: TJobQualityReportTableItem) => ( + <> + { onShow(record); }}>{intl.formatMessage({ id: 'jobs_task_report_detail' })} + + ), + }, + ]; + return ( +
+
+ +
+ + 报告时间 + + + + + +
+ +
+ + {intl.formatMessage({ id: 'quality_report_dashboard_score' })} + + + + + + {intl.formatMessage({ id: 'quality_report_dashboard_trend' })} + + + + + + + + {intl.formatMessage({ id: 'quality_report_dashboard_detail' })} + + + size="middle" + loading={loading} + rowKey="id" + columns={columns} + dataSource={tableData.list || []} + onChange={onPageChange} + pagination={{ + size: 'small', + total: tableData.total, + showSizeChanger: true, + current: pageParam.pageNumber, + pageSize: pageParam.pageSize, + pageSizeOptions: [5, 10, 20, 50, 100], + }} + /> + + + + + { + setisScheduleOpen(false); + }} + open={isScheduleOpen} + maskClosable={false} + > + { + setisScheduleOpen(false); + }} + /> + + + ); +}; + +export default QualityReportDashboard; diff --git a/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/columnExecutionResult.tsx b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/columnExecutionResult.tsx new file mode 100644 index 000000000..84a9ba9f6 --- /dev/null +++ b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/columnExecutionResult.tsx @@ -0,0 +1,94 @@ +/* eslint-disable react/no-danger */ +import React, { useState } from 'react'; +import { Table } from 'antd'; +import { ColumnsType } from 'antd/lib/table'; +import { + useWatch, +} from 'src/common'; +import { useIntl } from 'react-intl'; +import { $http } from '@/http'; +import { defaultRender } from '@/utils/helper'; +import {TJobQualityReportTableItem} from "@/type/JobQualityReport"; + +type InnerProps = { + [key: string]: any +} + +const Index = (props: InnerProps) => { + const intl = useIntl(); + const [loading, setLoading] = useState(false); + const [tableData, setTableData] = useState<{ list: TJobQualityReportTableItem[], total: number}>({ list: [], total: 0 }); + const [pageParams, setPageParams] = useState({ + pageNumber: 1, + pageSize: 10, + }); + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_metric_type' }), + dataIndex: 'metricName', + key: 'metricName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_check_formula' }), + dataIndex: 'resultFormulaFormat', + key: 'resultFormulaFormat', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + + { + title: intl.formatMessage({ id: 'jobs_task_score' }), + dataIndex: 'score', + key: 'score', + width: 200, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_check_result' }), + dataIndex: 'checkResult', + key: 'checkResult', + width: 200, + render: (text: string) => defaultRender(text, 300), + },{ + title: intl.formatMessage({ id: 'jobs_task_last_time' }), + dataIndex: 'executionTime', + key: 'executionTime', + width: 200, + render: (text: string) => defaultRender(text, 300), + } + ]; + const getData = async () => { + try { + setLoading(true); + const res = (await $http.get('/job/quality-report/listColumnExecution', { + reportId: props.record.id + })) || []; + setTableData({ + list: res || [], + total: 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + useWatch([], async () => { + getData(); + }, { immediate: true }); + // @ts-ignore + return ( +
+ + size="small" + rowKey="id" + columns={columns} + dataSource={tableData.list || []} + pagination={false} + /> +
+ ); +}; + +export default Index; diff --git a/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/index.tsx b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/index.tsx new file mode 100644 index 000000000..75f272ff8 --- /dev/null +++ b/datavines-ui/src/view/Main/HomeDetail/Dashboard/qualityReportDashboard/qualityReportDetailModal/index.tsx @@ -0,0 +1,225 @@ +/* eslint-disable react/no-danger */ +import React, { useRef, useState } from 'react'; +import { ModalProps, Table } from 'antd'; +import { ColumnsType } from 'antd/lib/table'; +import { + useModal, useImmutable, usePersistFn, useWatch, +} from 'src/common'; +import { useIntl } from 'react-intl'; +import { $http } from '@/http'; +import { defaultRender } from '@/utils/helper'; +import {TJobQualityReportTableItem} from "@/type/JobQualityReport"; +import ColumnExecutionResult from "./columnExecutionResult"; +import {TableColumnsType} from "antd/lib"; + +type InnerProps = { + [key: string]: any +} +const Inner = (props: InnerProps) => { + const intl = useIntl(); + const [loading, setLoading] = useState(false); + const [tableData, setTableData] = useState<{ list: TJobQualityReportTableItem[], total: number}>({ list: [], total: 0 }); + const [pageParams, setPageParams] = useState({ + pageNumber: 1, + pageSize: 10, + }); + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'databaseName', + key: 'databaseName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_column_name' }), + dataIndex: 'columnName', + key: 'columnName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_score' }), + dataIndex: 'score', + key: 'score', + width: 200, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_report_date' }), + dataIndex: 'reportDate', + key: 'reportDate', + width: 200, + render: (text: string) =>
{text || '--'}
, + } + ]; + const getData = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/quality-report/page', { + datasourceId: props.record.datasourceId, + schemaName : props.record.databaseName, + tableName : props.record.tableName, + reportDate: props.record.reportDate, + ...pageParams, + })) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + useWatch([pageParams], async () => { + getData(); + }, { immediate: true }); + const onChange = ({ current, pageSize }: any) => { + setPageParams({ + pageNumber: current, + pageSize, + }); + }; + + const expandedRowRender2 = (props: InnerProps) => { + const intl = useIntl(); + const [tableData, setTableData] = useState<{ list: TJobQualityReportTableItem[], total: number}>({ list: [], total: 0 }); + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'jobs_task_schema_name' }), + dataIndex: 'databaseName', + key: 'databaseName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_table_name' }), + dataIndex: 'tableName', + key: 'tableName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_column_name' }), + dataIndex: 'columnName', + key: 'columnName', + width: 200, + render: (text: string) => defaultRender(text, 300), + }, + { + title: intl.formatMessage({ id: 'jobs_task_score' }), + dataIndex: 'score', + key: 'score', + width: 200, + render: (text: string) =>
{text}
, + }, + { + title: intl.formatMessage({ id: 'jobs_task_report_date' }), + dataIndex: 'reportDate', + key: 'reportDate', + width: 200, + render: (text: string) =>
{text || '--'}
, + } + ]; + const getData = async () => { + try { + setLoading(true); + const res = (await $http.post('/job/quality-report/page', { + datasourceId: props.record.datasourceId, + schemaName : props.record.databaseName, + tableName : props.record.tableName, + ...pageParams, + })) || []; + setTableData({ + list: res?.records || [], + total: res.total || 0, + }); + } catch (error) { + } finally { + setLoading(false); + } + }; + useWatch([pageParams], async () => { + getData(); + }, { immediate: true }); + + // @ts-ignore + return ( +
+ + rowKey="id" + columns={columns} + dataSource={tableData.list || []} + /> +
+ ); + }; + + return ( +
+ + size="middle" + loading={loading} + rowKey="id" + columns={columns} + dataSource={tableData.list || []} + onChange={onChange} + scroll={{ + x: (columns.length) * 120, + }} + + expandable={{ + // eslint-disable-next-line react/no-unstable-nested-components + expandedRowRender: (record: TJobQualityReportTableItem) => ( + + ) + + }} + pagination={{ + size: 'small', + total: tableData.total, + showSizeChanger: true, + current: pageParams.pageNumber, + pageSize: pageParams.pageSize, + }} + /> +
+ ); +}; + +export const qualityReportDetailModal = (options: ModalProps) => { + const intl = useIntl(); + const recordRef = useRef(); + const onOk = usePersistFn(() => { + hide(); + }); + const { + Render, hide, show, ...rest + } = useModal({ + title: `${intl.formatMessage({ id: 'jobs_task_report_detail' })}`, + className: 'dv-modal-fullscreen', + footer: null, + width: '90%', + ...(options || {}), + afterClose() { + recordRef.current = null; + }, + onOk, + }); + return { + Render: useImmutable(() => ()), + show(record: any) { + recordRef.current = record; + show(record); + }, + ...rest, + }; +}; diff --git a/datavines-ui/src/view/Main/HomeDetail/JobExecutionLogs/index.tsx b/datavines-ui/src/view/Main/HomeDetail/JobExecutionLogs/index.tsx index 590bf116b..89b38e070 100644 --- a/datavines-ui/src/view/Main/HomeDetail/JobExecutionLogs/index.tsx +++ b/datavines-ui/src/view/Main/HomeDetail/JobExecutionLogs/index.tsx @@ -82,35 +82,35 @@ const JobExecutionLogs = ({ datasourceId }: TJobExecutionLogs) => { dataIndex: 'name', key: 'name', width: 300, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_schema_name' }), dataIndex: 'schemaName', key: 'schemaName', width: 150, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_table_name' }), dataIndex: 'tableName', key: 'tableName', width: 150, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_column_name' }), dataIndex: 'columnName', key: 'columnName', width: 150, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_metric_type' }), dataIndex: 'metricType', key: 'metricType', width: 150, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_type' }), @@ -301,8 +301,5 @@ const JobExecutionLogs = ({ datasourceId }: TJobExecutionLogs) => { ); - - - } export default JobExecutionLogs; \ No newline at end of file diff --git a/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsInstance.tsx b/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsInstance.tsx old mode 100755 new mode 100644 index 31af19e0c..d180b2501 --- a/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsInstance.tsx +++ b/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsInstance.tsx @@ -5,8 +5,8 @@ import { ColumnsType } from 'antd/lib/table'; import { useIntl } from 'react-intl'; import querystring from 'querystring'; import { TJobsInstanceTableData, TJobsInstanceTableItem } from '@/type/JobsInstance'; -import { Title, SearchForm } from '@/component'; -import { useMount, IF } from '@/common'; +import { SearchForm } from '@/component'; +import {IF, useWatch} from '@/common'; import { $http } from '@/http'; import { defaultRender } from '@/utils/helper'; import { useLogger } from './useLogger'; @@ -26,12 +26,12 @@ const JobsInstance = () => { pageSize: 10, }); const [qs] = useState(querystring.parse(window.location.href.split('?')[1] || '')); - const getData = async (values?: any, $pageParams?: any) => { + const getData = async (values: any = null) => { try { setLoading(true); const res = (await $http.post('/job/execution/page', { jobId: qs.jobId, - ...($pageParams || pageParams), + ...pageParams, ...(values || form.getFieldsValue()), })) || []; setTableData({ @@ -43,26 +43,25 @@ const JobsInstance = () => { setLoading(false); } }; - useMount(() => { + + useWatch([pageParams], () => { getData(); - }); + }, { immediate: true }); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars const onSearch = (_values: any) => { setPageParams({ ...pageParams, pageNumber: 1 }); - getData({ - ..._values, - pageNumber: 1, - }); + getData(); }; + const onChange = ({ current, pageSize }: any) => { setPageParams({ pageNumber: current, pageSize, }); - getData(null, { - pageNumber: current, - pageSize, - }); + getData(); }; + const onStop = async (record: TJobsInstanceTableItem) => { try { setLoading(true); @@ -88,35 +87,35 @@ const JobsInstance = () => { dataIndex: 'name', key: 'name', width: 300, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_schema_name' }), dataIndex: 'schemaName', key: 'schemaName', width: 100, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_table_name' }), dataIndex: 'tableName', key: 'tableName', width: 200, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_column_name' }), dataIndex: 'columnName', key: 'columnName', width: 200, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_metric_type' }), dataIndex: 'metricType', key: 'metricType', width: 200, - render: (text) => defaultRender(text, 300), + render: (text: string) => defaultRender(text, 300), }, { title: intl.formatMessage({ id: 'jobs_task_type' }), @@ -161,7 +160,7 @@ const JobsInstance = () => { width: 200, render: (text: string, record: TJobsInstanceTableItem) => ( <> - + { onStop(record); }}>{intl.formatMessage({ id: 'jobs_task_stop_btn' })} { onLog(record); }}>{intl.formatMessage({ id: 'jobs_task_log_btn' })} diff --git a/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsList.tsx b/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsList.tsx index a2d4aa079..69a5203a6 100644 --- a/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsList.tsx +++ b/datavines-ui/src/view/Main/HomeDetail/Jobs/JobsList.tsx @@ -30,22 +30,27 @@ const Jobs = ({ datasourceId }: TJobs) => { getData(); } }); + const { Render: RenderJobsModal, show: showJobsModal } = useAddEditJobsModal({ title: intl.formatMessage({ id: addType === 'quality' ? 'jobs_tabs_title' : 'jobs_tabs_comparison_title' }), afterClose() { getData(); }, }); + const { Render: RenderSLAsModal, show: showSLAsModal } = useSelectSLAsModal({ afterClose() { getData(); }, }); + const [tableData, setTableData] = useState({ list: [], total: 0 }); + const [pageParams, setPageParams] = useState({ pageNumber: 1, pageSize: 10, }); + const handleMenuClick: MenuProps['onClick'] = (e) => { setAddType(e.key); store.dispatch({ @@ -67,6 +72,7 @@ const Jobs = ({ datasourceId }: TJobs) => { }, }); }; + const menu = ( { /> ); - const transDateFormat = (datetime: any, format: string) => { if (datetime){ const day = dayjs(datetime) @@ -115,22 +120,22 @@ const Jobs = ({ datasourceId }: TJobs) => { setLoading(false); } }; + useWatch([pageParams], () => { getData(); }, { immediate: true }); - // const onSearch = () => { - // setPageParams({ ...pageParams, pageNumber: 1 }); - // }; + const onSearch = (_values: any) => { setPageParams({ ...pageParams, pageNumber: 1 }); - // getData() }; + const onChange = ({ current, pageSize }: any) => { setPageParams({ pageNumber: current, pageSize, }); }; + const onRun = async (record: TJobsTableItem) => { try { setLoading(true); @@ -142,6 +147,7 @@ const Jobs = ({ datasourceId }: TJobs) => { setLoading(false); } }; + const onEdit = (record: TJobsTableItem) => { const type = record.type === 'DATA_RECONCILIATION' ? 'comparison' : 'quality'; store.dispatch({ @@ -154,6 +160,7 @@ const Jobs = ({ datasourceId }: TJobs) => { record, }); }; + const onDelete = async (record: TJobsTableItem) => { try { setLoading(true); @@ -165,12 +172,15 @@ const Jobs = ({ datasourceId }: TJobs) => { setLoading(false); } }; + const onSLAs = (record: TJobsTableItem) => { showSLAsModal(record); }; + const onViewInstance = (record: TJobsTableItem) => { history.push(`${match.url}/instance?jobId=${record.id}`); }; + const onPreviewInstance = async (record: TJobsTableItem) => { showJobPreviewModal(record.id); }; diff --git a/datavines-ui/src/view/Main/HomeDetail/Jobs/components/Schedule/index.tsx b/datavines-ui/src/view/Main/HomeDetail/Jobs/components/Schedule/index.tsx index 4cffc67b2..2e92820a2 100644 --- a/datavines-ui/src/view/Main/HomeDetail/Jobs/components/Schedule/index.tsx +++ b/datavines-ui/src/view/Main/HomeDetail/Jobs/components/Schedule/index.tsx @@ -450,8 +450,8 @@ const Schedule: React.FC = ({ formRef, detail }) => { ); }; const ScheduleContainer = ({ - jobId, api = 'job', width, onSavaEnd, style = {}, isShowPush = false, -}: {jobId: string | number, api?:string;width?:string;onSavaEnd?:()=>void;style?:any;isShowPush?:boolean}) => { + jobId, api = 'job', width, onSavaEnd, style = {}, isShowPush = false, taskType = '-1' +}: {jobId: string | number, api?:string; width?:string; onSavaEnd?:()=>void; style?:any; isShowPush?:boolean; taskType: string }) => { const intl = useIntl(); const globalSetLoading = useLoading(); const [loading, setLoading] = useState(true); @@ -485,7 +485,13 @@ const ScheduleContainer = ({ }; const getData = async () => { try { - const res = await $http.get(`/${api}/schedule/${jobId}`); + let res = null; + if (taskType === '-1') { + res = await $http.get(`/${api}/schedule/${jobId}`); + } else { + res = await $http.get(`/${api}/schedule/${jobId}/${taskType}`); + } + if (res) { if (res.param) { res.param = JSON.parse(res.param); @@ -529,9 +535,17 @@ const ScheduleContainer = ({ }, [jobId]); const onSave = async (type:string = 'add') => { if (type === 'push') { - await $http.post('/catalog/refresh', { - datasourceId: jobId, - }); + if (taskType === '-1') { + await $http.post('/catalog/refresh', { + datasourceId: jobId + }); + } else { + await $http.post('/catalog/refresh', { + datasourceId: jobId, + taskType: taskType + }); + } + // eslint-disable-next-line no-unused-expressions onSavaEnd && onSavaEnd(); message.success(intl.formatMessage({ id: 'common_success' })); @@ -546,8 +560,9 @@ const ScheduleContainer = ({ entityUUID: jobId, ...params, }; - if (api === 'catalog/metadata') { + if (taskType !== '-1') { $params.dataSourceId = jobId; + $params.taskType = taskType; } if (detail?.id) { $params.id = detail?.id; diff --git a/datavines-ui/src/view/Main/HomeDetail/Jobs/useAddEditJobsModal/index.tsx b/datavines-ui/src/view/Main/HomeDetail/Jobs/useAddEditJobsModal/index.tsx index 537ab908d..fc99843c2 100644 --- a/datavines-ui/src/view/Main/HomeDetail/Jobs/useAddEditJobsModal/index.tsx +++ b/datavines-ui/src/view/Main/HomeDetail/Jobs/useAddEditJobsModal/index.tsx @@ -227,7 +227,7 @@ export const Inner = ({ - + diff --git a/datavines-ui/src/view/Main/HomeDetail/index.tsx b/datavines-ui/src/view/Main/HomeDetail/index.tsx old mode 100755 new mode 100644 diff --git a/mvnw b/mvnw old mode 100755 new mode 100644 diff --git a/scripts/sql/datavines-mysql-upgrade-0602.sql b/scripts/sql/datavines-mysql-upgrade-0602.sql new file mode 100644 index 000000000..7fcc90ec0 --- /dev/null +++ b/scripts/sql/datavines-mysql-upgrade-0602.sql @@ -0,0 +1,44 @@ +RENAME TABLE dv_catalog_metadata_fetch_command TO dv_common_task_command; +RENAME TABLE dv_catalog_metadata_fetch_task TO dv_common_task; +RENAME TABLE dv_catalog_metadata_fetch_task_schedule TO dv_common_task_schedule; + +ALTER TABLE dv_actual_values MODIFY COLUMN actual_value decimal(20,4) NULL COMMENT '实际值'; +ALTER TABLE dv_common_task ADD task_type varchar(128) NULL COMMENT '任务类型'; +ALTER TABLE dv_common_task_schedule ADD task_type varchar(128) NULL COMMENT '任务类型'; + +ALTER TABLE dv_job_execution_result MODIFY COLUMN actual_value decimal(20,4) DEFAULT NULL COMMENT '实际值'; +ALTER TABLE dv_job_execution_result MODIFY COLUMN expected_value decimal(20,4) DEFAULT NULL COMMENT '期望值'; +ALTER TABLE dv_job_execution_result ADD score decimal(20,4) DEFAULT 0 COMMENT '质量评分'; + +-- ---------------------------- +-- Table structure for dv_job_quality_report +-- ---------------------------- +DROP TABLE IF EXISTS `dv_job_quality_report`; +CREATE TABLE `dv_job_quality_report` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `datasource_id` bigint(20) DEFAULT NULL COMMENT '数据源ID', + `entity_level` varchar(128) DEFAULT NULL COMMENT '实体级别：DATASOURCE，DATABASE，TABLE，COLUMN', + `database_name` varchar(128) DEFAULT NULL COMMENT '数据库名称', + `table_name` varchar(128) DEFAULT NULL COMMENT '表名称', + `column_name` varchar(128) DEFAULT NULL COMMENT '列名称', + `score` decimal(20,4) DEFAULT NULL COMMENT '质量评分', + `report_date` date DEFAULT NULL COMMENT '报告日期', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='数据质量报告'; + +-- ---------------------------- +-- Table structure for dv_job_execution_result_report_rel +-- ---------------------------- +DROP TABLE IF EXISTS `dv_job_execution_result_report_rel`; +CREATE TABLE `dv_job_execution_result_report_rel` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `quality_report_id` bigint(20) NOT NULL COMMENT '质量报告ID', + `job_execution_result_id` bigint(20) NOT NULL COMMENT '作业执行结果ID', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `dv_execution_report_rel_un` (`job_execution_result_id`,`quality_report_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='质量报告和执行结果关联关系'; + diff --git a/scripts/sql/datavines-mysql.sql b/scripts/sql/datavines-mysql.sql index a445d6902..ace79b523 100644 --- a/scripts/sql/datavines-mysql.sql +++ b/scripts/sql/datavines-mysql.sql @@ -169,7 +169,7 @@ CREATE TABLE `dv_actual_values` ( `job_execution_id` bigint(20) DEFAULT NULL COMMENT '规则作业运行实例ID', `metric_name` varchar(255) DEFAULT NULL COMMENT '规则名称', `unique_code` varchar(255) DEFAULT NULL COMMENT '规则唯一编码', - `actual_value` double DEFAULT NULL COMMENT '实际值', + `actual_value` decimal(20,4) DEFAULT NULL COMMENT '实际值', `data_time` datetime DEFAULT NULL COMMENT '数据时间', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', @@ -177,12 +177,12 @@ CREATE TABLE `dv_actual_values` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='规则运行结果实际值'; -- ---------------------------- --- Table structure for dv_catalog_metadata_fetch_command +-- Table structure for dv_common_task_command -- ---------------------------- -DROP TABLE IF EXISTS `dv_catalog_metadata_fetch_command`; -CREATE TABLE `dv_catalog_metadata_fetch_command` ( +DROP TABLE IF EXISTS `dv_common_task_command`; +CREATE TABLE `dv_common_task_command` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, - `task_id` bigint(20) NOT NULL COMMENT '元数据抓取任务ID', + `task_id` bigint(20) NOT NULL COMMENT '通用任务ID', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', PRIMARY KEY (`id`) @@ -352,12 +352,13 @@ CREATE TABLE `dv_catalog_tag_category` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='标签分类'; -- ---------------------------- --- Table structure for dv_catalog_metadata_fetch_task +-- Table structure for dv_common_task -- ---------------------------- -DROP TABLE IF EXISTS `dv_catalog_metadata_fetch_task`; -CREATE TABLE `dv_catalog_metadata_fetch_task` ( +DROP TABLE IF EXISTS `dv_common_task`; +CREATE TABLE `dv_common_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, - `type` varchar(128) DEFAULT NULL COMMENT '类型', + `task_type` varchar(128) DEFAULT NULL COMMENT '任务类型', + `type` varchar(128) DEFAULT NULL COMMENT '粒度类型', `datasource_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '数据源ID', `database_name` varchar(128) DEFAULT NULL COMMENT '数据库', `table_name` varchar(128) DEFAULT NULL COMMENT '表', @@ -374,11 +375,12 @@ CREATE TABLE `dv_catalog_metadata_fetch_task` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='元数据抓取任务'; -- ---------------------------- --- Table structure for dv_catalog_metadata_fetch_task_schedule +-- Table structure for dv_common_task_schedule -- ---------------------------- -DROP TABLE IF EXISTS `dv_catalog_metadata_fetch_task_schedule`; -CREATE TABLE `dv_catalog_metadata_fetch_task_schedule` ( +DROP TABLE IF EXISTS `dv_common_task_schedule`; +CREATE TABLE `dv_common_task_schedule` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_type` varchar(128) DEFAULT NULL COMMENT '任务类型', `type` varchar(255) NOT NULL COMMENT '调度类型', `param` text COMMENT '调度参数', `datasource_id` bigint(20) NOT NULL COMMENT '数据源ID', @@ -576,12 +578,13 @@ CREATE TABLE `dv_job_execution_result` ( `database_name` varchar(128) DEFAULT NULL COMMENT '数据库名称', `table_name` varchar(128) DEFAULT NULL COMMENT '表名称', `column_name` varchar(128) DEFAULT NULL COMMENT '列名称', - `actual_value` double DEFAULT NULL COMMENT '实际值', - `expected_value` double DEFAULT NULL COMMENT '期望值', + `actual_value` decimal(20,4) DEFAULT NULL COMMENT '实际值', + `expected_value` decimal(20,4) DEFAULT NULL COMMENT '期望值', `expected_type` varchar(255) DEFAULT NULL COMMENT '期望值类型', `result_formula` varchar(255) DEFAULT NULL COMMENT '计算结果公式', `operator` varchar(255) DEFAULT NULL COMMENT '比较符', - `threshold` double DEFAULT NULL COMMENT '阈值', + `threshold` decimal(20,4) DEFAULT NULL COMMENT '阈值', + `score` decimal(20,4) DEFAULT 0 COMMENT '质量评分', `state` int(2) NOT NULL DEFAULT '0' COMMENT '结果 1:success/2:fail', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', @@ -589,6 +592,38 @@ CREATE TABLE `dv_job_execution_result` ( UNIQUE KEY `execution_id_un` (`job_execution_id`,`metric_unique_key`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='规则作业运行结果'; +-- ---------------------------- +-- Table structure for dv_job_quality_report +-- ---------------------------- +DROP TABLE IF EXISTS `dv_job_quality_report`; +CREATE TABLE `dv_job_quality_report` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `datasource_id` bigint(20) DEFAULT NULL COMMENT '数据源ID', + `entity_level` varchar(128) DEFAULT NULL COMMENT '实体级别：DATASOURCE，DATABASE，TABLE，COLUMN', + `database_name` varchar(128) DEFAULT NULL COMMENT '数据库名称', + `table_name` varchar(128) DEFAULT NULL COMMENT '表名称', + `column_name` varchar(128) DEFAULT NULL COMMENT '列名称', + `score` decimal(20,4) DEFAULT NULL COMMENT '质量评分', + `report_date` date DEFAULT NULL COMMENT '报告日期', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='数据质量报告'; + +-- ---------------------------- +-- Table structure for dv_job_execution_result_report_rel +-- ---------------------------- +DROP TABLE IF EXISTS `dv_job_execution_result_report_rel`; +CREATE TABLE `dv_job_execution_result_report_rel` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `quality_report_id` bigint(20) NOT NULL COMMENT '质量报告ID', + `job_execution_result_id` bigint(20) NOT NULL COMMENT '作业执行结果ID', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `dv_execution_report_rel_un` (`job_execution_result_id`,`quality_report_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='质量报告和执行结果关联关系'; + -- ---------------------------- -- Table structure for dv_job_issue_rel -- ----------------------------