[Feature][Server] Add data quality report feature (#404)

datavines-common/src/main/java/io/datavines/common/utils/DateUtils.java

@@ -43,6 +43,7 @@ public class DateUtils {
     public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss";
 
     public static final String YYYYMMDD = "yyyyMMdd";
+    public static final String YYYY_MM_DD = "yyyy-MM-dd";
 
     /**
      * date to local datetime

datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConfigBuilder.java

@@ -67,7 +67,7 @@ protected InputParam getPropertiesInput(boolean isEn) {
         return getInputParam("properties",
                 isEn ? "properties" : "参数",
                 isEn ? "please enter properties,like key=value&key1=value1" : "请填入参数，格式为key=value&key1=value1", 2, null,
-                "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true");
+                "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true&allowPublicKeyRetrieval=true");
     }
 
     @Override

datavines-core/src/main/java/io/datavines/core/constant/DataVinesConstants.java

@@ -83,4 +83,6 @@ public class DataVinesConstants {
     public static final String SPARK = "spark";
 
     public static final String LOCALE_LANGUAGE_COOKIE = "language";
+
+    public static final String COMMON_TASK_TYPE = "commonTaskType";
 }

datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java

@@ -20,15 +20,15 @@
 
 public class LoggerFactory {
 
-    private static Logger logger;
+    private static ThreadLocal<Logger> logger = new ThreadLocal<>();
 
     public static void setLogger(Logger newLogger){
-        logger = newLogger;
+        logger.set(newLogger);
     }
 
     public static Logger getLogger(Class clazz) {
-        if (logger != null) {
-            return logger;
+        if (logger.get() != null) {
+            return logger.get();
         } else {
             return org.slf4j.LoggerFactory.getLogger(clazz);
         }

datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java

@@ -69,7 +69,6 @@ protected List<SourceConfig> getSourceConfigs() throws DataVinesException {
                     connectorParameterMap.putAll(metricInputParameter);
                     connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap);
 
-
                     metricInputParameter.put(DATABASE_NAME, metricInputParameter.get(DATABASE));
                     metricInputParameter.put(TABLE_NAME, metricInputParameter.get(TABLE));
                     metricInputParameter.put(COLUMN_NAME, metricInputParameter.get(COLUMN));
@@ -140,7 +139,6 @@ protected List<SourceConfig> getSourceConfigs() throws DataVinesException {
                     connectorParameterMap.put(TABLE, table);
                     connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap);
 
-
                     String outputTable = metricInputParameter.get(TABLE2);
                     connectorParameterMap.put(OUTPUT_TABLE, outputTable);
                     connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver());

datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricDirectionType.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.datavines.metric.api;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import lombok.Getter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum MetricDirectionType {
+    /**
+     * 0-positive
+     * 1-negative
+     */
+    POSITIVE(0,"positive"),
+    NEGATIVE(1,"negative");
+
+    MetricDirectionType(int code, String description) {
+        this.code = code;
+        this.description = description;
+    }
+
+    @Getter
+    private final int code;
+    private final String description;
+
+    @JsonValue
+    public String getDescription() {
+        return description;
+    }
+
+    private static final Map<Integer, MetricDirectionType> VALUES_MAP = new HashMap<>();
+
+    static {
+        for (MetricDirectionType type : MetricDirectionType.values()) {
+            VALUES_MAP.put(type.code,type);
+        }
+    }
+
+    public static MetricDirectionType of(Integer code) {
+        if (VALUES_MAP.containsKey(code)) {
+            return VALUES_MAP.get(code);
+        }
+        throw new IllegalArgumentException("invalid code : " + code);
+    }
+
+}

datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricExecutionResult.java

@@ -22,22 +22,23 @@
 import org.apache.commons.collections4.MapUtils;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.Map;
 
 @Data
 public class MetricExecutionResult implements Serializable {
 
     private static final long serialVersionUID = -1L;
 
-    private Double actualValue;
+    private BigDecimal actualValue;
 
-    private Double expectedValue;
+    private BigDecimal expectedValue;
 
     private String resultFormula;
 
     private String operator;
 
-    private Double threshold;
+    private BigDecimal threshold;
 
     private String expectedType;
 
@@ -62,11 +63,11 @@ public MetricExecutionResult(Map<String, Object> dataMap) {
         }
 
         if (dataMap.get("actual_value") != null) {
-            actualValue = Double.valueOf(String.valueOf(dataMap.get("actual_value")).trim());
+            actualValue = new BigDecimal(String.valueOf(dataMap.get("actual_value")).trim());
         }
 
         if (dataMap.get("expected_value") != null) {
-            expectedValue = Double.valueOf(String.valueOf(dataMap.get("expected_value")).trim());
+            expectedValue = new BigDecimal(String.valueOf(dataMap.get("expected_value")).trim());
         }
 
         if (dataMap.get("result_formula") != null) {
@@ -78,7 +79,7 @@ public MetricExecutionResult(Map<String, Object> dataMap) {
         }
 
         if (dataMap.get("threshold") != null) {
-            threshold = Double.valueOf(String.valueOf(dataMap.get("threshold")).trim());
+            threshold = new BigDecimal(String.valueOf(dataMap.get("threshold")).trim());
         }
 
         if (dataMap.get("metric_name") != null) {

datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java

@@ -32,8 +32,8 @@ public class MetricValidator {
      */
     public static boolean isSuccess(MetricExecutionResult executionResult) {
 
-        Double actualValue = executionResult.getActualValue();
-        Double expectedValue = executionResult.getExpectedValue();
+        BigDecimal actualValue = executionResult.getActualValue();
+        BigDecimal expectedValue = executionResult.getExpectedValue();
 
         OperatorType operatorType = OperatorType.of(StringUtils.trim(executionResult.getOperator()));
         ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class)
@@ -44,26 +44,36 @@ public static boolean isSuccess(MetricExecutionResult executionResult) {
                 executionResult.getThreshold());
     }
 
-    private static boolean getCompareResult(OperatorType operatorType, Double srcValue, Double targetValue) {
+    public static BigDecimal getQualityScore(MetricExecutionResult executionResult, boolean isSuccess) {
+        BigDecimal actualValue = executionResult.getActualValue();
+        BigDecimal expectedValue = executionResult.getExpectedValue();
+
+        ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class)
+                .getOrCreatePlugin(executionResult.getResultFormula());
+        SqlMetric metric = PluginLoader.getPluginLoader(SqlMetric.class)
+                .getOrCreatePlugin(executionResult.getMetricName());
+
+        return resultFormula.getScore(actualValue, expectedValue, isSuccess, metric.getDirectionType());
+    }
+
+    private static boolean getCompareResult(OperatorType operatorType, BigDecimal srcValue, BigDecimal targetValue) {
         if (srcValue == null || targetValue == null) {
             return false;
         }
 
-        BigDecimal src = BigDecimal.valueOf(srcValue);
-        BigDecimal target = BigDecimal.valueOf(targetValue);
         switch (operatorType) {
             case EQ:
-                return src.compareTo(target) == 0;
+                return srcValue.compareTo(targetValue) == 0;
             case LT:
-                return src.compareTo(target) <= -1;
+                return srcValue.compareTo(targetValue) <= -1;
             case LTE:
-                return src.compareTo(target) == 0 || src.compareTo(target) <= -1;
+                return srcValue.compareTo(targetValue) == 0 || srcValue.compareTo(targetValue) <= -1;
             case GT:
-                return src.compareTo(target) >= 1;
+                return srcValue.compareTo(targetValue) >= 1;
             case GTE:
-                return src.compareTo(target) == 0 || src.compareTo(target) >= 1;
+                return srcValue.compareTo(targetValue) == 0 || srcValue.compareTo(targetValue) >= 1;
             case NE:
-                return src.compareTo(target) != 0;
+                return srcValue.compareTo(targetValue) != 0;
             default:
                 return true;
         }

datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java

@@ -18,6 +18,8 @@
 
 import io.datavines.spi.SPI;
 
+import java.math.BigDecimal;
+
 @SPI
 public interface ResultFormula {
 
@@ -29,7 +31,9 @@ default String getNameByLanguage(boolean isEn) {
         return isEn ? getName() : getZhName();
     }
 
-    Double getResult(Double actualValue, Double expectedValue);
+    BigDecimal getResult(BigDecimal actualValue, BigDecimal expectedValue);
+
+    BigDecimal getScore(BigDecimal actualValue, BigDecimal expectedValue, boolean isSuccess, MetricDirectionType direction);
 
     String getResultFormat(boolean isEn);
 

datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java

@@ -92,4 +92,8 @@ default boolean supportMultiple() {
     default List<Map<String,Object>> getMetricParameter(Map<String,Object> metricParameter) {
         return Collections.singletonList(metricParameter);
     }
+
+    default MetricDirectionType getDirectionType() {
+        return MetricDirectionType.POSITIVE;
+    }
 }

datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java

@@ -48,7 +48,7 @@ public String getKey(Map<String,String> inputParameter) {
     public String getExecuteSql(Map<String,String> inputParameter) {
         String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY);
         String newKey = "expected_value_" + uniqueKey;
-        return sql.toString().replace("expected_value",newKey);
+        return sql.toString().replace("expected_value", newKey);
     }
 
     @Override

datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java

@@ -18,10 +18,11 @@
 
 import io.datavines.common.enums.DataVinesDataType;
 import io.datavines.metric.api.MetricDimension;
+import io.datavines.metric.api.MetricDirectionType;
 import io.datavines.metric.api.MetricType;
 import io.datavines.metric.plugin.base.BaseSingleTableColumn;
 
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -66,6 +67,11 @@ public void prepare(Map<String, String> config) {
 
     @Override
     public List<DataVinesDataType> suitableType() {
-        return Arrays.asList(DataVinesDataType.STRING_TYPE);
+        return Collections.singletonList(DataVinesDataType.STRING_TYPE);
+    }
+
+    @Override
+    public MetricDirectionType getDirectionType() {
+        return MetricDirectionType.NEGATIVE;
     }
 }

datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java

@@ -18,6 +18,7 @@
 
 import io.datavines.common.enums.DataVinesDataType;
 import io.datavines.metric.api.MetricDimension;
+import io.datavines.metric.api.MetricDirectionType;
 import io.datavines.metric.api.MetricType;
 import io.datavines.metric.plugin.base.BaseSingleTableColumn;
 import org.apache.commons.lang3.StringUtils;
@@ -75,4 +76,9 @@ public void prepare(Map<String, String> config) {
     public List<DataVinesDataType> suitableType() {
         return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE);
     }
+
+    @Override
+    public MetricDirectionType getDirectionType() {
+        return MetricDirectionType.NEGATIVE;
+    }
 }

datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java

@@ -19,6 +19,7 @@
 import io.datavines.common.enums.DataVinesDataType;
 import io.datavines.metric.api.ConfigItem;
 import io.datavines.metric.api.MetricDimension;
+import io.datavines.metric.api.MetricDirectionType;
 import io.datavines.metric.api.MetricType;
 import io.datavines.metric.plugin.base.BaseSingleTableColumn;
 
@@ -74,4 +75,9 @@ public void prepare(Map<String, String> config) {
     public List<DataVinesDataType> suitableType() {
         return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE);
     }
+
+    @Override
+    public MetricDirectionType getDirectionType() {
+        return MetricDirectionType.NEGATIVE;
+    }
 }

datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java

@@ -68,6 +68,7 @@ public void prepare(Map<String, String> config) {
         }
         super.prepare(config);
     }
+
     @Override
     public List<DataVinesDataType> suitableType() {
         return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE);

datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java

@@ -19,6 +19,7 @@
 import io.datavines.common.enums.DataVinesDataType;
 import io.datavines.metric.api.ConfigItem;
 import io.datavines.metric.api.MetricDimension;
+import io.datavines.metric.api.MetricDirectionType;
 import io.datavines.metric.api.MetricType;
 import io.datavines.metric.plugin.base.BaseSingleTableColumn;
 
@@ -74,4 +75,9 @@ public List<DataVinesDataType> suitableType() {
         return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE);
     }
 
+    @Override
+    public MetricDirectionType getDirectionType() {
+        return MetricDirectionType.NEGATIVE;
+    }
+
 }

datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java

@@ -59,7 +59,6 @@ public boolean isInvalidateItemsCanOutput() {
 
     @Override
     public void prepare(Map<String, String> config) {
-
         if (config.containsKey("column")) {
             filters.add(" ${column} is not null ");
         }

datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java

@@ -18,6 +18,7 @@
 
 import io.datavines.common.enums.DataVinesDataType;
 import io.datavines.metric.api.MetricDimension;
+import io.datavines.metric.api.MetricDirectionType;
 import io.datavines.metric.api.MetricType;
 import io.datavines.metric.plugin.base.BaseSingleTableColumn;
 
@@ -69,4 +70,9 @@ public void prepare(Map<String, String> config) {
     public List<DataVinesDataType> suitableType() {
         return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE);
     }
+
+    @Override
+    public MetricDirectionType getDirectionType() {
+        return MetricDirectionType.NEGATIVE;
+    }
 }