[Fix][Server] Fix quality dashboard error

datavane · Nov 17, 2023 · f4f9e39 · f4f9e39
1 parent 90a30cd
commit f4f9e39
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 9 deletions.
diff --git a/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java b/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java
@@ -119,11 +119,12 @@ public class ConfigConstants {
 
     public static final String UNIX_TIMESTAMP = "unix_timestamp";
 
-
     public static final String STRING_TYPE = "string_type";
 
     public static final String IF_FUNCTION_KEY = "if_function_key";
 
+    public static final String IF_CASE_KEY = "if_case_key";
+
     public static final String LIMIT_TOP_50_KEY = "limit_top_50_key";
 
     public static final String LIMIT_KEY = "limit_key";

diff --git a/datavines-common/src/main/java/io/datavines/common/enums/DataVinesDataType.java b/datavines-common/src/main/java/io/datavines/common/enums/DataVinesDataType.java
@@ -119,7 +119,9 @@ public static DataVinesDataType getType(String type) {
         }
 
         type = type.toLowerCase();
-        if (type.contains("int") || "decimal".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "double".equalsIgnoreCase(type) || type.contains("number")) {
+        if (type.contains("int") || "decimal".equalsIgnoreCase(type)
+                || "float".equalsIgnoreCase(type) || "double".equalsIgnoreCase(type)
+                || type.contains("number") || type.contains("numeric") || type.contains("real") || type.contains("serial")) {
             return NUMERIC_TYPE;
         } else if (type.contains("char") || "blob".equalsIgnoreCase(type) || type.contains("text")) {
             return STRING_TYPE;

diff --git a/...ins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java b/...ins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java
@@ -38,6 +38,7 @@ public Map<String, String> getDialectKeyMap() {
         dialectKeyMap.put(IF_FUNCTION_KEY, "if");
         dialectKeyMap.put(LIMIT_TOP_50_KEY, " limit 50");
         dialectKeyMap.put(LENGTH_KEY, "length(${column})");
+        dialectKeyMap.put(IF_CASE_KEY, "if(${column} is null, 'NULL', cast(${column} as ${string_type}))");
         return dialectKeyMap;
     }
 

diff --git a/...s-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java b/...s-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java
@@ -28,6 +28,7 @@ public Map<String, String> getDialectKeyMap() {
         dialectKeyMap.put(REGEX_KEY, "${column} ~ '${regexp}'");
         dialectKeyMap.put(NOT_REGEX_KEY, "${column} !~ '${regexp}'");
         dialectKeyMap.put(LENGTH_KEY, "length(${column}::text)");
+        dialectKeyMap.put(IF_CASE_KEY, "case when ${column} is null then 'NULL' else cast(${column} as ${string_type}) end ");
         return dialectKeyMap;
     }
 

diff --git a/...nes-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java b/...nes-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java
@@ -18,10 +18,7 @@
 
 import java.util.Map;
 
-import static io.datavines.common.ConfigConstants.STRING_TYPE;
-import static io.datavines.common.ConfigConstants.IF_FUNCTION_KEY;
-import static io.datavines.common.ConfigConstants.LIMIT_TOP_50_KEY;
-import static io.datavines.common.ConfigConstants.LENGTH_KEY;
+import static io.datavines.common.ConfigConstants.*;
 
 public class SqlServerDialect extends JdbcDialect {
 
@@ -32,6 +29,7 @@ public Map<String, String> getDialectKeyMap() {
         dialectKeyMap.put(LENGTH_KEY, "len(${column})");
         dialectKeyMap.put(IF_FUNCTION_KEY, "iif");
         dialectKeyMap.put(LIMIT_TOP_50_KEY, " OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY");
+        dialectKeyMap.put(IF_CASE_KEY, "iif(${column} is null, 'NULL', cast(${column} as ${string_type}))");
         return dialectKeyMap;
     }
 

diff --git a/...nes-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java b/...nes-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java
@@ -71,8 +71,8 @@ public ExecuteSql getActualValue(Map<String,String> inputParameter) {
         ExecuteSql executeSql = new ExecuteSql();
         executeSql.setResultTable("invalidate_count_"+uniqueKey);
         StringBuilder actualValueSql = new StringBuilder();
-        actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select ${if_function_key}(${column} is null, 'NULL', cast(${column} as ${string_type})) as k, count(1) as count from ${table}");
-        if (filters.size() > 0) {
+        actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select ${if_case_key} as k, count(1) as count from ${table}");
+        if (!filters.isEmpty()) {
             actualValueSql.append(" where ").append(String.join(" and ", filters));
         }
 

diff --git a/...er/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java b/...er/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java
@@ -304,10 +304,39 @@ public List<MetricExecutionDashBoard> getMetricExecutionDashBoard(Long jobId, St
     public List<JobExecutionAggItem> getJobExecutionAggPie(JobExecutionDashboardParam dashboardParam) {
         List<String> statusList = new ArrayList<>(Arrays.asList("6","7"));
 
+        String startDateStr = "";
+        String endDateStr = "";
+        if (StringUtils.isEmpty(dashboardParam.getStartTime()) && StringUtils.isEmpty(dashboardParam.getEndTime())) {
+            startDateStr = DateUtils.format(DateUtils.addDays(new Date(), -5),"yyyy-MM-dd");
+            endDateStr = DateUtils.format(DateUtils.addDays(new Date(), +1),"yyyy-MM-dd");
+        } else {
+            if (StringUtils.isEmpty(dashboardParam.getEndTime()) && StringUtils.isNotEmpty(dashboardParam.getStartTime())) {
+                startDateStr = dashboardParam.getStartTime().substring(0,10);
+                Date startDate = DateUtils.stringToDate(dashboardParam.getStartTime());
+                endDateStr = DateUtils.format(DateUtils.addDays(startDate,7),"yyyy-MM-dd");
+            } else if (StringUtils.isEmpty(dashboardParam.getStartTime()) && StringUtils.isNotEmpty(dashboardParam.getEndTime())) {
+                endDateStr = dashboardParam.getEndTime().substring(0,10);
+                Date endDate = DateUtils.stringToDate(dashboardParam.getEndTime());
+                startDateStr = DateUtils.format(DateUtils.addDays(endDate,-6),"yyyy-MM-dd");
+            } else {
+                Date endDate = DateUtils.parse(dashboardParam.getEndTime(), YYYY_MM_DD_HH_MM_SS);
+                Date startDate = DateUtils.parse(dashboardParam.getStartTime(), YYYY_MM_DD_HH_MM_SS);
+                long days = DateUtils.diffDays(endDate,startDate);
+                if (days > 7) {
+                    endDate = DateUtils.addDays(startDate, 7);
+                }
+                startDateStr = DateUtils.format(startDate,"yyyy-MM-dd");
+                endDateStr = DateUtils.format(endDate,"yyyy-MM-dd");
+            }
+        }
+        startDateStr += " 00:00:00";
+        endDateStr += " 23:59:59";
+
         List<JobExecutionAggItem> items =
                 baseMapper.getJobExecutionAggPie(dashboardParam.getDatasourceId(), dashboardParam.getMetricType(),
                         dashboardParam.getSchemaName(), dashboardParam.getTableName(), dashboardParam.getColumnName(),
-                        dashboardParam.getStartTime(), dashboardParam.getEndTime());
+                        startDateStr, endDateStr);
+
         if (CollectionUtils.isEmpty(items)) {
             return new ArrayList<>();
         }
@@ -399,6 +428,8 @@ public JobExecutionTrendBar getJobExecutionTrendBar(JobExecutionDashboardParam d
             currentDate = currentDate.plusDays(1);
         }
 
+        startDateStr += " 00:00:00";
+        endDateStr += " 23:59:59";
         List<JobExecutionTrendBarItem> trendBars = baseMapper.getJobExecutionTrendBar(dashboardParam.getDatasourceId(),
                 dashboardParam.getMetricType(), dashboardParam.getSchemaName(), dashboardParam.getTableName(), dashboardParam.getColumnName(),
                 startDateStr, endDateStr);