Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix][Connector] Fix postgre sql connector if function bug #285

Merged
merged 1 commit into from
Nov 17, 2023
Merged

[Fix][Connector] Fix postgre sql connector if function bug #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion datavines-common/src/main/java/io/datavines/common/ConfigConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,12 @@ public class ConfigConstants {

public static final String UNIX_TIMESTAMP = "unix_timestamp";


public static final String STRING_TYPE = "string_type";

public static final String IF_FUNCTION_KEY = "if_function_key";

public static final String IF_CASE_KEY = "if_case_key";

public static final String LIMIT_TOP_50_KEY = "limit_top_50_key";

public static final String LIMIT_KEY = "limit_key";
Expand Down
4 changes: 3 additions & 1 deletion datavines-common/src/main/java/io/datavines/common/enums/DataVinesDataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,9 @@ public static DataVinesDataType getType(String type) {
}

type = type.toLowerCase();
if (type.contains("int") || "decimal".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "double".equalsIgnoreCase(type) || type.contains("number")) {
if (type.contains("int") || "decimal".equalsIgnoreCase(type)
|| "float".equalsIgnoreCase(type) || "double".equalsIgnoreCase(type)
|| type.contains("number") || type.contains("numeric") || type.contains("real") || type.contains("serial")) {
return NUMERIC_TYPE;
} else if (type.contains("char") || "blob".equalsIgnoreCase(type) || type.contains("text")) {
return STRING_TYPE;
Expand Down
1 change: 1 addition & 0 deletions ...ins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public Map<String, String> getDialectKeyMap() {
dialectKeyMap.put(IF_FUNCTION_KEY, "if");
dialectKeyMap.put(LIMIT_TOP_50_KEY, " limit 50");
dialectKeyMap.put(LENGTH_KEY, "length(${column})");
dialectKeyMap.put(IF_CASE_KEY, "if(${column} is null, 'NULL', cast(${column} as ${string_type}))");
return dialectKeyMap;
}

Expand Down
1 change: 1 addition & 0 deletions ...s-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public Map<String, String> getDialectKeyMap() {
dialectKeyMap.put(REGEX_KEY, "${column} ~ '${regexp}'");
dialectKeyMap.put(NOT_REGEX_KEY, "${column} !~ '${regexp}'");
dialectKeyMap.put(LENGTH_KEY, "length(${column}::text)");
dialectKeyMap.put(IF_CASE_KEY, "case when ${column} is null then 'NULL' else cast(${column} as ${string_type}) end ");
return dialectKeyMap;
}

Expand Down
6 changes: 2 additions & 4 deletions ...nes-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,7 @@

import java.util.Map;

import static io.datavines.common.ConfigConstants.STRING_TYPE;
import static io.datavines.common.ConfigConstants.IF_FUNCTION_KEY;
import static io.datavines.common.ConfigConstants.LIMIT_TOP_50_KEY;
import static io.datavines.common.ConfigConstants.LENGTH_KEY;
import static io.datavines.common.ConfigConstants.*;

public class SqlServerDialect extends JdbcDialect {

Expand All @@ -32,6 +29,7 @@ public Map<String, String> getDialectKeyMap() {
dialectKeyMap.put(LENGTH_KEY, "len(${column})");
dialectKeyMap.put(IF_FUNCTION_KEY, "iif");
dialectKeyMap.put(LIMIT_TOP_50_KEY, " OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY");
dialectKeyMap.put(IF_CASE_KEY, "iif(${column} is null, 'NULL', cast(${column} as ${string_type}))");
return dialectKeyMap;
}

Expand Down
4 changes: 2 additions & 2 deletions ...nes-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ public ExecuteSql getActualValue(Map<String,String> inputParameter) {
ExecuteSql executeSql = new ExecuteSql();
executeSql.setResultTable("invalidate_count_"+uniqueKey);
StringBuilder actualValueSql = new StringBuilder();
actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select ${if_function_key}(${column} is null, 'NULL', cast(${column} as ${string_type})) as k, count(1) as count from ${table}");
if (filters.size() > 0) {
actualValueSql.append("select concat(k, '\001', cast(count as ${string_type})) as actual_value_").append(uniqueKey).append(" from (select ${if_case_key} as k, count(1) as count from ${table}");
if (!filters.isEmpty()) {
actualValueSql.append(" where ").append(String.join(" and ", filters));
}

Expand Down
33 changes: 32 additions & 1 deletion ...er/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,10 +304,39 @@ public List<MetricExecutionDashBoard> getMetricExecutionDashBoard(Long jobId, St
public List<JobExecutionAggItem> getJobExecutionAggPie(JobExecutionDashboardParam dashboardParam) {
List<String> statusList = new ArrayList<>(Arrays.asList("6","7"));

String startDateStr = "";
String endDateStr = "";
if (StringUtils.isEmpty(dashboardParam.getStartTime()) && StringUtils.isEmpty(dashboardParam.getEndTime())) {
startDateStr = DateUtils.format(DateUtils.addDays(new Date(), -5),"yyyy-MM-dd");
endDateStr = DateUtils.format(DateUtils.addDays(new Date(), +1),"yyyy-MM-dd");
} else {
if (StringUtils.isEmpty(dashboardParam.getEndTime()) && StringUtils.isNotEmpty(dashboardParam.getStartTime())) {
startDateStr = dashboardParam.getStartTime().substring(0,10);
Date startDate = DateUtils.stringToDate(dashboardParam.getStartTime());
endDateStr = DateUtils.format(DateUtils.addDays(startDate,7),"yyyy-MM-dd");
} else if (StringUtils.isEmpty(dashboardParam.getStartTime()) && StringUtils.isNotEmpty(dashboardParam.getEndTime())) {
endDateStr = dashboardParam.getEndTime().substring(0,10);
Date endDate = DateUtils.stringToDate(dashboardParam.getEndTime());
startDateStr = DateUtils.format(DateUtils.addDays(endDate,-6),"yyyy-MM-dd");
} else {
Date endDate = DateUtils.parse(dashboardParam.getEndTime(), YYYY_MM_DD_HH_MM_SS);
Date startDate = DateUtils.parse(dashboardParam.getStartTime(), YYYY_MM_DD_HH_MM_SS);
long days = DateUtils.diffDays(endDate,startDate);
if (days > 7) {
endDate = DateUtils.addDays(startDate, 7);
}
startDateStr = DateUtils.format(startDate,"yyyy-MM-dd");
endDateStr = DateUtils.format(endDate,"yyyy-MM-dd");
}
}
startDateStr += " 00:00:00";
endDateStr += " 23:59:59";

List<JobExecutionAggItem> items =
baseMapper.getJobExecutionAggPie(dashboardParam.getDatasourceId(), dashboardParam.getMetricType(),
dashboardParam.getSchemaName(), dashboardParam.getTableName(), dashboardParam.getColumnName(),
dashboardParam.getStartTime(), dashboardParam.getEndTime());
startDateStr, endDateStr);

if (CollectionUtils.isEmpty(items)) {
return new ArrayList<>();
}
Expand Down Expand Up @@ -399,6 +428,8 @@ public JobExecutionTrendBar getJobExecutionTrendBar(JobExecutionDashboardParam d
currentDate = currentDate.plusDays(1);
}

startDateStr += " 00:00:00";
endDateStr += " 23:59:59";
List<JobExecutionTrendBarItem> trendBars = baseMapper.getJobExecutionTrendBar(dashboardParam.getDatasourceId(),
dashboardParam.getMetricType(), dashboardParam.getSchemaName(), dashboardParam.getTableName(), dashboardParam.getColumnName(),
startDateStr, endDateStr);
Expand Down
Loading