diff --git a/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java b/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java index c22f719b5..07ffffa78 100644 --- a/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java +++ b/datavines-common/src/main/java/io/datavines/common/ConfigConstants.java @@ -22,6 +22,7 @@ public class ConfigConstants { public static final String CONNECTOR_TYPE = "connector_type"; public static final String DATASOURCE_ID = "datasource_id"; public static final String TABLE = "table"; + public static final String VIEW_SQL = "view_sql"; public static final String TABLE_ALIAS = "table_alias"; public static final String TABLE2_ALIAS = "table2_alias"; public static final String TABLE_ALIAS_COLUMNS = "table_alias_columns"; @@ -37,6 +38,7 @@ public class ConfigConstants { public static final String ACTUAL_NAME = "actual_name"; public static final String ACTUAL_EXECUTE_SQL = "actual_execute_sql"; public static final String ACTUAL_AGGREGATE_SQL = "actual_aggregate_sql"; + public static final String ACTUAL_CUSTOM_SQL = "actual_custom_sql"; public static final String EXPECTED_NAME = "expected_name"; public static final String EXPECTED_TYPE = "expected_type"; public static final String EXPECTED_TABLE = "expected_table"; @@ -74,9 +76,12 @@ public class ConfigConstants { public static final String ERROR_DATA_FILE_NAME = "error_data_file_name"; public static final String VALIDATE_RESULT_DATA_DIR = "validate_result_data_dir"; public static final String INVALIDATE_ITEM_CAN_OUTPUT = "invalidate_item_can_output"; + + public static final String ERROR_DATA_STORAGE_ID = "error_data_storage_id"; public static final String ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE = "error_data_output_to_datasource_database"; public static final String DATASOURCE = "datasource"; public static final String DATABASE = "database"; + public static final String SID = "sid"; public static final String DATABASE_NAME = "database_name"; public static final String TABLE_NAME = "table_name"; public static final String COLUMN_NAME = "column_name"; diff --git a/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java b/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java index a9b5530ee..f015a6bab 100644 --- a/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java +++ b/datavines-common/src/main/java/io/datavines/common/config/enums/SourceType.java @@ -23,9 +23,9 @@ public enum SourceType { /** - * 0 normal - * 1 invalidate items - * 2 actual value + * 0 source + * 1 target + * 2 metadata **/ SOURCE(0, "source"), TARGET(1, "target"), diff --git a/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java b/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java index 4c5d40e4a..48d45f8e2 100644 --- a/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java +++ b/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/BaseJdbcDataSourceInfo.java @@ -16,12 +16,16 @@ */ package io.datavines.common.datasource.jdbc; +import io.datavines.common.utils.Md5Utils; import io.datavines.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.DriverManager; +import java.util.Map; + +import static io.datavines.common.ConfigConstants.*; /** * data source base class @@ -30,26 +34,26 @@ public abstract class BaseJdbcDataSourceInfo { private static final Logger logger = LoggerFactory.getLogger(BaseJdbcDataSourceInfo.class); - protected final JdbcConnectionInfo jdbcConnectionInfo; + protected final Map param; - public BaseJdbcDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - this.jdbcConnectionInfo = jdbcConnectionInfo; + public BaseJdbcDataSourceInfo(Map param) { + this.param = param; } public String getUser() { - return jdbcConnectionInfo.getUser(); + return param.get(USER); } public String getPassword() { - return jdbcConnectionInfo.getPassword(); + return param.get(PASSWORD); } public String getHost() { - return jdbcConnectionInfo.getHost(); + return param.get(HOST); } public String getPort() { - return jdbcConnectionInfo.getPort(); + return param.get(PORT); } public String getValidationQuery() { @@ -59,19 +63,19 @@ public String getValidationQuery() { public abstract String getAddress(); public String getCatalog() { - return jdbcConnectionInfo.getCatalog(); + return param.get(CATALOG); } public String getDatabase() { - return jdbcConnectionInfo.getDatabase(); + return param.get(DATABASE); } public String getSchema() { - return jdbcConnectionInfo.getSchema(); + return param.get(SCHEMA); } public String getProperties() { - return jdbcConnectionInfo.getProperties(); + return param.get(PROPERTIES); } /** @@ -159,6 +163,21 @@ public void loadClass() { } public String getUniqueKey() { - return jdbcConnectionInfo.getUniqueKey(); + return Md5Utils.getMd5(paramToString(), false); + } + + public String paramToString() { + return getHost().trim() + + "&" + getPort() + + "&" + getOrEmpty(getCatalog()) + + "&" + getOrEmpty(getDatabase()) + + "&" + getOrEmpty(getUser()) + + "&" + getOrEmpty(getPassword()) + + "&" + getProperties(); } + + private String getOrEmpty(String keyword) { + return StringUtils.isNotEmpty(keyword)? keyword.trim() : ""; + } + } diff --git a/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java b/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java index e9f87f9be..b32cf3a42 100644 --- a/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java +++ b/datavines-common/src/main/java/io/datavines/common/datasource/jdbc/IJdbcDataSourceInfo.java @@ -16,7 +16,9 @@ */ package io.datavines.common.datasource.jdbc; +import java.util.Map; + public interface IJdbcDataSourceInfo { - BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo); + BaseJdbcDataSourceInfo getDatasourceInfo(Map param); } diff --git a/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java b/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java index 595a0eae6..9e6e29954 100644 --- a/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java +++ b/datavines-common/src/main/java/io/datavines/common/entity/QueryColumn.java @@ -18,8 +18,12 @@ import lombok.Data; +import java.io.Serializable; + @Data -public class QueryColumn { +public class QueryColumn implements Serializable { + + private static final long serialVersionUID = -2398995167525051291L; private String name; diff --git a/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java b/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java index f5b0533e2..6550ad16b 100644 --- a/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java +++ b/datavines-common/src/main/java/io/datavines/common/enums/ExecutionStatus.java @@ -16,11 +16,11 @@ */ package io.datavines.common.enums; -import java.util.HashMap; - import com.baomidou.mybatisplus.annotation.EnumValue; import lombok.Getter; +import java.util.HashMap; + /** * running status for workflow and task nodes * @@ -55,7 +55,7 @@ public enum ExecutionStatus { WAITING_THREAD(10, "waiting thread", "等待线程"), WAITING_SUMMIT(11, "waiting_summit","待提交"); - ExecutionStatus(int code, String description,String zhDescription){ + ExecutionStatus(int code, String description, String zhDescription){ this.code = code; this.description = description; this.zhDescription = zhDescription; diff --git a/datavines-common/src/main/java/io/datavines/common/enums/JobType.java b/datavines-common/src/main/java/io/datavines/common/enums/JobType.java index be02c1f40..7b0ec686b 100644 --- a/datavines-common/src/main/java/io/datavines/common/enums/JobType.java +++ b/datavines-common/src/main/java/io/datavines/common/enums/JobType.java @@ -41,6 +41,7 @@ public enum JobType { @EnumValue private final int code; + private final String description; private final String zhDescription; diff --git a/datavines-common/src/main/java/io/datavines/common/enums/OperatorType.java b/datavines-common/src/main/java/io/datavines/common/enums/OperatorType.java index f7a45a219..3fffc7913 100644 --- a/datavines-common/src/main/java/io/datavines/common/enums/OperatorType.java +++ b/datavines-common/src/main/java/io/datavines/common/enums/OperatorType.java @@ -40,7 +40,7 @@ public enum OperatorType { GTE(4,"gte",">="), NE(5,"neq","!="); - OperatorType(int code, String description,String symbol) { + OperatorType(int code, String description, String symbol) { this.code = code; this.description = description; this.symbol = symbol; diff --git a/datavines-common/src/main/java/io/datavines/common/utils/ParameterUtils.java b/datavines-common/src/main/java/io/datavines/common/utils/ParameterUtils.java index 85a3bc243..a84038be1 100644 --- a/datavines-common/src/main/java/io/datavines/common/utils/ParameterUtils.java +++ b/datavines-common/src/main/java/io/datavines/common/utils/ParameterUtils.java @@ -38,7 +38,6 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; - private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -74,7 +73,6 @@ public static String convertParameterPlaceholders(String parameterString, Map statements = statementSplitter.splitStatements(script); + + if (CollectionUtils.isEmpty(statements)) { + return null; + } + + for (int i=0; i splitStatements(String body); +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ColumnLineage.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ColumnLineage.java new file mode 100644 index 000000000..c3fc41e7b --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ColumnLineage.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class ColumnLineage { + + private List inputColumns; + + private List outputColumns; +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/MetaDataConstants.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/MetaDataConstants.java new file mode 100644 index 000000000..76c6f6973 --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/MetaDataConstants.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api.entity; + +public class MetaDataConstants { + + public final static String UNKNOWN_STATEMENT_TYPE = "UNKNOWN"; +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ScriptMetadata.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ScriptMetadata.java new file mode 100644 index 000000000..cb412dd7e --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/ScriptMetadata.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api.entity; + +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; +import java.util.ArrayList; +import java.util.List; + +@Data +public class ScriptMetadata { + + private String script; + + private List statementMetadataList; + + public void addStatementMetadata(StatementMetadata statementMetadata) { + if (CollectionUtils.isEmpty(statementMetadataList)) { + statementMetadataList = new ArrayList<>(); + } + statementMetadataList.add(statementMetadata); + } +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadata.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadata.java new file mode 100644 index 000000000..a4f387434 --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadata.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api.entity; + +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class StatementMetadata { + + private int statementIndex; + + private String statementText; + + private StatementMetadataFragment statementMetadataFragment; + + private LocalDateTime statementParseStartTime; + + private LocalDateTime statementParseEndTime; +} diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadataFragment.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadataFragment.java new file mode 100644 index 000000000..35174d4c1 --- /dev/null +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/entity/StatementMetadataFragment.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.api.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class StatementMetadataFragment { + + private List inputTables; + + private List outputTables; + + private List columnLineageList; +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnector.java index 495046ccb..da9f6d079 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnector.java @@ -17,12 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class ClickHouseConnector extends JdbcConnector { @@ -30,8 +30,8 @@ public ClickHouseConnector(DataSourceClient dataSourceClient) { super(dataSourceClient); } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new ClickHouseDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new ClickHouseDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDataSourceInfo.java index 0c8ada42f..6a78dcb5c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class ClickHouseDataSourceInfo extends BaseJdbcDataSourceInfo { - public ClickHouseDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public ClickHouseDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseExecutor.java index b08682abd..e7987ddda 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class ClickHouseExecutor extends BaseJdbcExecutor { public ClickHouseExecutor(DataSourceClient jdbcDataSourceClient) { @@ -27,7 +28,7 @@ public ClickHouseExecutor(DataSourceClient jdbcDataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new ClickHouseDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new ClickHouseDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnector.java index 7d98515ba..8540364b0 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnector.java @@ -17,7 +17,6 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.param.ConnectorResponse; import io.datavines.common.param.TestConnectionRequestParam; import io.datavines.common.utils.JSONUtils; @@ -25,6 +24,7 @@ import io.datavines.connector.api.DataSourceClient; import java.sql.*; +import java.util.Map; public class DatabendConnector extends JdbcConnector { @@ -48,14 +48,14 @@ public ResultSet getMetadataDatabases(Connection connection) throws SQLException } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new DatabendDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new DatabendDataSourceInfo(param); } @Override public ConnectorResponse testConnect(TestConnectionRequestParam param) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(param.getDataSourceParam(), JdbcConnectionInfo.class); - BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(jdbcConnectionInfo); + Map paramMap = JSONUtils.toMap(param.getDataSourceParam()); + BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(paramMap); dataSourceInfo.loadClass(); try (Connection con = DriverManager.getConnection(dataSourceInfo.getJdbcUrl(), dataSourceInfo.getUser(), StringUtils.isEmpty(dataSourceInfo.getPassword()) ? null : dataSourceInfo.getPassword())) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendDataSourceInfo.java index 16ee03f2e..5056f487c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class DatabendDataSourceInfo extends BaseJdbcDataSourceInfo { - public DatabendDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public DatabendDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendExecutor.java index 5b898893d..87bc0e270 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendExecutor.java @@ -17,16 +17,17 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class DatabendExecutor extends BaseJdbcExecutor { public DatabendExecutor(DataSourceClient jdbcDataSourceClient) { super(jdbcDataSourceClient); } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new DatabendDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new DatabendDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnector.java index 6746955d5..1aeb3d449 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnector.java @@ -17,13 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class DmConnector extends JdbcConnector { @@ -32,8 +32,8 @@ public DmConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new DmDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new DmDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmDataSourceInfo.java index 080724aca..6e4555399 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmDataSourceInfo.java @@ -17,15 +17,16 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + public class DmDataSourceInfo extends BaseJdbcDataSourceInfo { private final Logger logger = LoggerFactory.getLogger(DmDataSourceInfo.class); - public DmDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public DmDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmExecutor.java index d4344b354..dd11e1288 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class DmExecutor extends BaseJdbcExecutor{ public DmExecutor(DataSourceClient jdbcDataSourceClient) { @@ -27,7 +28,7 @@ public DmExecutor(DataSourceClient jdbcDataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new DmDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new DmDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnector.java index a87bc1100..3c6c5619b 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnector.java @@ -16,7 +16,6 @@ */ package io.datavines.connector.plugin; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.datasource.jdbc.entity.ColumnInfo; import io.datavines.common.datasource.jdbc.entity.TableColumnInfo; import io.datavines.common.datasource.jdbc.entity.TableInfo; @@ -27,12 +26,16 @@ import io.datavines.common.utils.JSONUtils; import io.datavines.common.utils.StringUtils; import io.datavines.connector.api.DataSourceClient; +import org.apache.commons.collections4.MapUtils; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import static io.datavines.common.ConfigConstants.CATALOG; public class DorisConnector extends MysqlConnector { @@ -45,12 +48,12 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); List tableList = null; ResultSet tables; @@ -90,20 +93,19 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept public ConnectorResponse getColumns(GetColumnsRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); TableColumnInfo tableColumnInfo = null; try { String catalog; String schema; - if (StringUtils.isNotEmpty(jdbcConnectionInfo.getCatalog())) { - catalog = jdbcConnectionInfo.getCatalog(); + if (StringUtils.isNotEmpty(paramMap.get(CATALOG))) { + catalog = paramMap.get(CATALOG); schema = param.getDataBase(); } else { catalog = null; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisDataSourceInfo.java index 16c59a8e9..6c5b23afc 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisDataSourceInfo.java @@ -16,13 +16,14 @@ */ package io.datavines.connector.plugin; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.utils.StringUtils; +import java.util.Map; + public class DorisDataSourceInfo extends MysqlDataSourceInfo { - public DorisDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public DorisDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisExecutor.java index 6bcb6004a..2e0c73590 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class DorisExecutor extends MysqlExecutor { public DorisExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public DorisExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new DorisDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new DorisDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java index 34b346cf9..6d7ddcb7a 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java @@ -64,4 +64,14 @@ public ConfigBuilder getConfigBuilder() { public DataSourceClient getDataSourceClient() { return null; } + + @Override + public StatementSplitter getStatementSplitter() { + return null; + } + + @Override + public StatementParser getStatementParser() { + return null; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnector.java index 900c227b2..42f010d55 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnector.java @@ -17,12 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class HiveConnector extends JdbcConnector { @@ -31,8 +31,8 @@ public HiveConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new HiveDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new HiveDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDataSourceInfo.java index 1e280ee5a..25f4d68b5 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class HiveDataSourceInfo extends BaseJdbcDataSourceInfo { - public HiveDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public HiveDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveExecutor.java index e1e97b34c..cfbe2c7d3 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveExecutor.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.datasource.jdbc.utils.HiveSqlUtils; import io.datavines.common.entity.ListWithQueryColumn; import io.datavines.connector.api.DataSourceClient; import org.springframework.jdbc.core.JdbcTemplate; +import java.util.Map; + public class HiveExecutor extends BaseJdbcExecutor { public HiveExecutor(DataSourceClient dataSourceClient) { @@ -30,8 +31,8 @@ public HiveExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new HiveDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new HiveDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnector.java index 413d60ba1..16a32329c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnector.java @@ -17,12 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class ImpalaConnector extends JdbcConnector { @@ -31,8 +31,8 @@ public ImpalaConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new ImpalaDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new ImpalaDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaDataSourceInfo.java index 49c167ef2..cef57d351 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaDataSourceInfo.java @@ -17,16 +17,17 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + public class ImpalaDataSourceInfo extends BaseJdbcDataSourceInfo { private final Logger logger = LoggerFactory.getLogger(ImpalaDataSourceInfo.class); - public ImpalaDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public ImpalaDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaExecutor.java index 96efe4305..29c124ba5 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class ImpalaExecutor extends BaseJdbcExecutor { public ImpalaExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public ImpalaExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new ImpalaDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new ImpalaDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java index ac9ea5ec2..7ab6a9514 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/AbstractJdbcConnectorFactory.java @@ -44,4 +44,14 @@ public ConfigBuilder getConfigBuilder() { public DataSourceClient getDataSourceClient() { return new JdbcDataSourceClient(); } + + @Override + public StatementSplitter getStatementSplitter() { + return new DefaultStatementSplitter(); + } + + @Override + public StatementParser getStatementParser() { + return new DefaultStatementParser(); + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/BaseJdbcExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/BaseJdbcExecutor.java index 0fda4460a..dc20bc03c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/BaseJdbcExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/BaseJdbcExecutor.java @@ -24,10 +24,12 @@ import io.datavines.common.utils.JSONUtils; import io.datavines.connector.api.DataSourceClient; import io.datavines.connector.api.Executor; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; import java.sql.SQLException; +import java.util.Map; public abstract class BaseJdbcExecutor implements Executor, IJdbcDataSourceInfo { @@ -46,10 +48,12 @@ public ConnectorResponse queryForPage(ExecuteRequestParam param) throws SQLExcep ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { + throw new SQLException("jdbc datasource param is no validate"); + } JdbcTemplate jdbcTemplate = dataSourceClient.getJdbcTemplate( - JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(jdbcConnectionInfo))); + JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(paramMap))); String sql = param.getScript(); if (StringUtils.isEmpty(sql)) { @@ -68,9 +72,13 @@ public ConnectorResponse queryForOne(ExecuteRequestParam param) throws SQLExcept ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { + throw new SQLException("jdbc datasource param is no validate"); + } + JdbcTemplate jdbcTemplate = dataSourceClient.getJdbcTemplate( - JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(jdbcConnectionInfo))); + JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(paramMap))); String sql = param.getScript() + " limit 1"; if (StringUtils.isEmpty(sql)) { @@ -88,10 +96,12 @@ public ConnectorResponse queryForOne(ExecuteRequestParam param) throws SQLExcept public ConnectorResponse queryForList(ExecuteRequestParam param) throws Exception { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { + throw new SQLException("jdbc datasource param is no validate"); + } JdbcTemplate jdbcTemplate = dataSourceClient.getJdbcTemplate( - JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(jdbcConnectionInfo))); + JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(paramMap))); String sql = param.getScript(); if (StringUtils.isEmpty(sql)) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementParser.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementParser.java new file mode 100644 index 000000000..327e7247f --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementParser.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.connector.api.StatementParser; +import io.datavines.connector.api.entity.StatementMetadataFragment; + +public class DefaultStatementParser implements StatementParser { + + @Override + public StatementMetadataFragment parseStatement(String statement) { + return null; + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java new file mode 100644 index 000000000..f2404f08c --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/DefaultStatementSplitter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.connector.plugin; + +import io.datavines.common.utils.StringUtils; +import io.datavines.connector.api.StatementSplitter; + +import java.util.Arrays; +import java.util.List; + +public class DefaultStatementSplitter implements StatementSplitter { + + @Override + public List splitStatements(String body) { + + String DELIMITER = ";"; + return StringUtils.isEmpty(body) ? null : Arrays.asList(body.split(DELIMITER)); + } +} diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java index a1647dfed..54a9f11c9 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-jdbc/src/main/java/io/datavines/connector/plugin/JdbcConnector.java @@ -27,6 +27,7 @@ import io.datavines.connector.api.Connector; import io.datavines.common.datasource.jdbc.utils.JdbcDataSourceUtils; import io.datavines.connector.api.DataSourceClient; +import org.apache.commons.collections4.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,9 +35,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; -import static io.datavines.common.ConfigConstants.HOST; -import static io.datavines.common.ConfigConstants.PORT; +import static io.datavines.common.ConfigConstants.*; public abstract class JdbcConnector implements Connector, IJdbcDataSourceInfo { @@ -60,22 +61,22 @@ public JdbcConnector(DataSourceClient dataSourceClient) { this.dataSourceClient = dataSourceClient; } - protected Connection getConnection(String dataSourceParam, JdbcConnectionInfo jdbcConnectionInfo) throws SQLException { - return dataSourceClient.getConnection(JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(jdbcConnectionInfo))); + protected Connection getConnection(String dataSourceParam, Map param) throws SQLException { + return dataSourceClient.getConnection(JdbcDataSourceInfoManager.getDatasourceInfo(dataSourceParam, getDatasourceInfo(param))); } @Override public ConnectorResponse getDatabases(GetDatabasesRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } List databaseList = new ArrayList<>(); - if (StringUtils.isEmptyOrNullStr(jdbcConnectionInfo.getDatabase())) { - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + if (StringUtils.isEmptyOrNullStr(paramMap.get(DATABASE))) { + Connection connection = getConnection(dataSourceParam, paramMap); ResultSet rs = getMetadataDatabases(connection); while (rs.next()) { @@ -83,7 +84,7 @@ public ConnectorResponse getDatabases(GetDatabasesRequestParam param) throws SQL } JdbcDataSourceUtils.releaseConnection(connection); } else { - databaseList.add(new DatabaseInfo(jdbcConnectionInfo.getDatabase(), DATABASE)); + databaseList.add(new DatabaseInfo(paramMap.get(DATABASE), DATABASE)); } builder.result(databaseList); @@ -96,12 +97,12 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); List tableList = null; ResultSet tables; @@ -111,12 +112,12 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept String catalog; String schema; - if (StringUtils.isNotEmpty(jdbcConnectionInfo.getCatalog())) { - catalog = jdbcConnectionInfo.getCatalog(); + if (StringUtils.isNotEmpty(paramMap.get(CATALOG))) { + catalog = paramMap.get(CATALOG); schema = param.getDatabase(); } else { catalog = param.getDatabase(); - schema = StringUtils.isEmptyOrNullStr(jdbcConnectionInfo.getSchema()) ? null : jdbcConnectionInfo.getSchema(); + schema = StringUtils.isEmptyOrNullStr(paramMap.get(SCHEMA)) ? null : paramMap.get(SCHEMA); } tableList = new ArrayList<>(); @@ -152,24 +153,24 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept public ConnectorResponse getColumns(GetColumnsRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); TableColumnInfo tableColumnInfo = null; try { String catalog; String schema; - if (StringUtils.isNotEmpty(jdbcConnectionInfo.getCatalog())) { - catalog = jdbcConnectionInfo.getCatalog(); + if (StringUtils.isNotEmpty(paramMap.get(CATALOG))) { + catalog = paramMap.get(CATALOG); schema = param.getDataBase(); } else { catalog = param.getDataBase(); - schema = StringUtils.isEmptyOrNullStr(jdbcConnectionInfo.getSchema()) ? null : jdbcConnectionInfo.getSchema(); + schema = StringUtils.isEmptyOrNullStr(paramMap.get(SCHEMA)) ? null : paramMap.get(SCHEMA); } String tableName = param.getTable(); @@ -195,8 +196,8 @@ public ConnectorResponse getPartitions(ConnectorRequestParam param) { @Override public ConnectorResponse testConnect(TestConnectionRequestParam param) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(param.getDataSourceParam(), JdbcConnectionInfo.class); - BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(jdbcConnectionInfo); + Map paramMap = JSONUtils.toMap(param.getDataSourceParam()); + BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(paramMap); dataSourceInfo.loadClass(); try (Connection con = DriverManager.getConnection(dataSourceInfo.getJdbcUrl(), dataSourceInfo.getUser(), dataSourceInfo.getPassword())) { diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbClientManager.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbClientManager.java index b8683926e..10ff937fa 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbClientManager.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbClientManager.java @@ -26,10 +26,10 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -37,17 +37,17 @@ public class MongodbClientManager { private final Logger logger = LoggerFactory.getLogger(MongodbClientManager.class); - private final LoadingCache mongoClientLoadingCache = CacheBuilder.newBuilder() + private final LoadingCache, MongoClient> mongoClientLoadingCache = CacheBuilder.newBuilder() .expireAfterAccess(5, TimeUnit.MINUTES) - .removalListener((RemovalListener) notification -> { + .removalListener((RemovalListener , MongoClient>) notification -> { notification.getValue().close(); }) - .build(new CacheLoader() { + .build(new CacheLoader, MongoClient>() { @Override - public MongoClient load(JdbcConnectionInfo jdbcConnectionInfo) { + public MongoClient load(Map param) { try { - BaseJdbcDataSourceInfo dataSourceInfo = new MongodbDataSourceInfo(jdbcConnectionInfo); + BaseJdbcDataSourceInfo dataSourceInfo = new MongodbDataSourceInfo(param); MongoClientSettings.Builder options = MongoClientSettings.builder(); String url = dataSourceInfo.getJdbcUrl(); ConnectionString connectionString = new ConnectionString(url); @@ -69,11 +69,11 @@ public static MongodbClientManager getInstance() { return MongodbClientManager.Singleton.INSTANCE; } - public MongoClient getMongoClient(JdbcConnectionInfo jdbcConnectionInfo) { + public MongoClient getMongoClient(Map param) { MongoClient mongoClient = null; try { - mongoClient = mongoClientLoadingCache.get(jdbcConnectionInfo); + mongoClient = mongoClientLoadingCache.get(param); } catch (ExecutionException e) { logger.error(e.toString(), e); } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java index ae0596464..e6b7014a6 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnector.java @@ -30,6 +30,7 @@ import io.datavines.common.utils.JSONUtils; import io.datavines.common.utils.StringUtils; import io.datavines.connector.api.Connector; +import org.apache.commons.collections4.MapUtils; import org.apache.curator.shaded.com.google.common.collect.Streams; import org.bson.Document; import org.bson.types.Binary; @@ -89,15 +90,15 @@ public ConnectorResponse getDatabases(GetDatabasesRequestParam param) throws SQL } private MongoClient getMongoClient(String dataSourceParam) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - return getMongoClient(jdbcConnectionInfo); + Map paramMap = JSONUtils.toMap(dataSourceParam); + return getMongoClient(paramMap); } - private MongoClient getMongoClient(JdbcConnectionInfo jdbcConnectionInfo) { + private MongoClient getMongoClient(Map param) { MongodbClientManager instance = MongodbClientManager.getInstance(); - return instance.getMongoClient(jdbcConnectionInfo); + return instance.getMongoClient(param); } @Override @@ -106,12 +107,12 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept String dataSourceParam = param.getDataSourceParam(); String dataBase = param.getDatabase(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - MongoClient mongoClient = getMongoClient(jdbcConnectionInfo); + MongoClient mongoClient = getMongoClient(paramMap); MongoDatabase database = mongoClient.getDatabase(dataBase); MongoIterable collectionList = database.listCollectionNames(); List tableInfos = new ArrayList<>(); @@ -129,15 +130,15 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept public ConnectorResponse getColumns(GetColumnsRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } String dataBase = param.getDataBase(); String table = param.getTable(); - MongoClient mongoClient = getMongoClient(jdbcConnectionInfo); + MongoClient mongoClient = getMongoClient(paramMap); MongoDatabase db = mongoClient.getDatabase(dataBase); Document sortDoc = new Document("_id", -1); Document doc = db.getCollection(table) @@ -193,10 +194,10 @@ private String guessFieldType(Object value) { @Override public ConnectorResponse testConnect(TestConnectionRequestParam param) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(param.getDataSourceParam(), JdbcConnectionInfo.class); + Map paramMap = JSONUtils.toMap(param.getDataSourceParam()); try { - MongoClient mongoClient = getMongoClient(jdbcConnectionInfo); + MongoClient mongoClient = getMongoClient(paramMap); if (mongoClient == null) { return ConnectorResponse.builder().status(ConnectorResponse.Status.SUCCESS).result(false).build(); } @@ -210,7 +211,7 @@ public ConnectorResponse testConnect(TestConnectionRequestParam param) { @Override public List keyProperties() { - return Arrays.asList(HOST, PORT, DATABASE); + return Arrays.asList(HOST, PORT); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbDataSourceInfo.java index 61af9ea0e..f5741e1ec 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbDataSourceInfo.java @@ -17,16 +17,14 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.utils.StringUtils; -import static io.datavines.common.ConfigConstants.*; -import static io.datavines.common.ConfigConstants.PROPERTIES; +import java.util.Map; public class MongodbDataSourceInfo extends BaseJdbcDataSourceInfo { - public MongodbDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public MongodbDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbExecutor.java index 002c0056f..f64460f82 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class MongodbExecutor extends BaseJdbcExecutor { public MongodbExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public MongodbExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new MongodbDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new MongodbDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java index 333775261..fea64a741 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnector.java @@ -17,13 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class MysqlConnector extends JdbcConnector { @@ -32,8 +32,8 @@ public MysqlConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new MysqlDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new MysqlDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDataSourceInfo.java index dd5e4f9f3..3372857a5 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class MysqlDataSourceInfo extends BaseJdbcDataSourceInfo { - public MysqlDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public MysqlDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlExecutor.java index 91aab1564..7d532b0ce 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class MysqlExecutor extends BaseJdbcExecutor { public MysqlExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public MysqlExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new MysqlDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new MysqlDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConfigBuilder.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConfigBuilder.java index 8fe1745ef..04919c4ae 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConfigBuilder.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConfigBuilder.java @@ -16,16 +16,48 @@ */ package io.datavines.connector.plugin; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.datavines.common.param.form.PluginParams; import io.datavines.common.param.form.Validate; import io.datavines.common.param.form.type.InputParam; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class OracleConfigBuilder extends JdbcConfigBuilder { -public class OracleConfigBuilder extends JdbcConfigBuilder{ @Override - protected InputParam getSchemaInput(boolean isEn) { - return getInputParam("schema", - isEn ? "schema" : "模式", - isEn ? "please enter schema" : "请填入模式", 1, - Validate.newBuilder().setRequired(true).setMessage(isEn ? "please enter schema" : "请填入模式").build(), null); + public String build(boolean isEn) { + List params = new ArrayList<>(); + params.add(getHostInput(isEn)); + params.add(getPortInput(isEn)); + params.add(getSID(isEn)); + params.add(getUserInput(isEn)); + params.add(getPasswordInput(isEn)); + params.add(getPropertiesInput(isEn)); + params.addAll(getOtherParams(isEn)); + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + String result = null; + + try { + result = mapper.writeValueAsString(params); + } catch (JsonProcessingException e) { + log.error("json parse error : ", e); + } + + return result; + } + + private InputParam getSID(boolean isEn) { + return getInputParam("sid", + isEn ? "sid" : "SID", + isEn ? "please enter sid" : "请填入SID", 1, + Validate.newBuilder().setRequired(true).setMessage(isEn ? "please enter sid" : "请填入SID").build(), null); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnector.java index cb7da562b..38279041e 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnector.java @@ -17,43 +17,43 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; -public class OracleConnector extends JdbcConnector{ +public class OracleConnector extends JdbcConnector { public OracleConnector(DataSourceClient dataSourceClient) { super(dataSourceClient); } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new OracleDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new OracleDataSourceInfo(param); } @Override public ResultSet getMetadataColumns(DatabaseMetaData metaData, String catalog, String schema, String tableName, String columnName) throws SQLException { - return metaData.getColumns(null,schema.toUpperCase(), tableName, columnName); + return metaData.getColumns(null,catalog.toUpperCase(), tableName, columnName); } @Override public ResultSet getMetadataTables(DatabaseMetaData metaData, String catalog, String schema) throws SQLException { - return metaData.getTables(null,schema.toUpperCase(), null, TABLE_TYPES); + return metaData.getTables(null,catalog.toUpperCase(), null, TABLE_TYPES); } @Override protected ResultSet getPrimaryKeys(DatabaseMetaData metaData, String catalog, String schema, String tableName) throws SQLException { - return metaData.getPrimaryKeys(null, schema.toUpperCase(), tableName); + return metaData.getPrimaryKeys(null, catalog.toUpperCase(), tableName); } @Override public ResultSet getMetadataDatabases(Connection connection) throws SQLException { java.sql.Statement stmt = connection.createStatement(); - return stmt.executeQuery("select NAME AS Database FROM V$database"); + return stmt.executeQuery("SELECT username AS Database FROM all_users"); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java index 20a624e8b..7d30e5e94 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorParameterConverter.java @@ -20,19 +20,16 @@ import java.util.Map; -import static io.datavines.common.ConfigConstants.HOST; -import static io.datavines.common.ConfigConstants.PORT; -import static io.datavines.common.ConfigConstants.DATABASE; -import static io.datavines.common.ConfigConstants.PROPERTIES; +import static io.datavines.common.ConfigConstants.*; public class OracleConnectorParameterConverter extends JdbcConnectorParameterConverter{ @Override protected String getUrl(Map parameter) { - String url = String.format("jdbc:oracle:thin:@%s:%s:%s", + String url = String.format("jdbc:oracle:thin:@//%s:%s/%s", parameter.get(HOST), parameter.get(PORT), - parameter.get(DATABASE)); + parameter.get(SID)); String properties = (String)parameter.get(PROPERTIES); if (StringUtils.isNotEmpty(properties)) { url += "?" + properties; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDataSourceInfo.java index b49232de3..bc77ab786 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleDataSourceInfo.java @@ -17,17 +17,21 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; +import io.datavines.common.utils.StringUtils; + +import java.util.Map; + +import static io.datavines.common.ConfigConstants.SID; public class OracleDataSourceInfo extends BaseJdbcDataSourceInfo { - public OracleDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo){ - super(jdbcConnectionInfo); + public OracleDataSourceInfo(Map param){ + super(param); } @Override public String getAddress() { - return "jdbc:oracle:thin:@" + getHost() + ":" + getPort(); + return "jdbc:oracle:thin:@//" + getHost() + ":" + getPort(); } @Override @@ -49,4 +53,24 @@ protected String getSeparator() { public String getValidationQuery() { return "Select 1 FROM DUAL"; } + + @Override + public String getJdbcUrl() { + StringBuilder jdbcUrl = new StringBuilder(getAddress()); + appendSid(jdbcUrl); + return jdbcUrl.toString(); + } + + /** + * append sid + * @param jdbcUrl jdbc url + */ + protected void appendSid(StringBuilder jdbcUrl) { + if (StringUtils.isNotEmpty(param.get(SID))) { + if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { + jdbcUrl.append("/"); + } + jdbcUrl.append(param.get(SID)); + } + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleExecutor.java index 535d741c9..4b65a8ce3 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class OracleExecutor extends BaseJdbcExecutor { public OracleExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public OracleExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new OracleDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new OracleDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnector.java index 82c8cd45f..c24a81b8a 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnector.java @@ -17,13 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class PostgreSqlConnector extends JdbcConnector { @@ -32,8 +32,8 @@ public PostgreSqlConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new PostgreSqlDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new PostgreSqlDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDataSourceInfo.java index 6f4cb6a31..00deaa113 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlDataSourceInfo.java @@ -17,17 +17,18 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + public class PostgreSqlDataSourceInfo extends BaseJdbcDataSourceInfo { private final Logger logger = LoggerFactory.getLogger(PostgreSqlDataSourceInfo.class); - public PostgreSqlDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public PostgreSqlDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlExecutor.java index 9665da006..8430bb040 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class PostgreSqlExecutor extends BaseJdbcExecutor { public PostgreSqlExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public PostgreSqlExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new PostgreSqlDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new PostgreSqlDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnector.java index b20b1299b..dc835047d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnector.java @@ -17,11 +17,9 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.param.ConnectorResponse; import io.datavines.common.param.TestConnectionRequestParam; -import io.datavines.common.param.form.Validate; -import io.datavines.common.param.form.type.InputParam; + import io.datavines.common.utils.JSONUtils; import io.datavines.common.utils.StringUtils; import io.datavines.connector.api.DataSourceClient; @@ -29,6 +27,7 @@ import java.sql.*; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import static io.datavines.common.ConfigConstants.PASSWORD; @@ -52,14 +51,15 @@ public ResultSet getMetadataDatabases(Connection connection) throws SQLException } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new PrestoDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new PrestoDataSourceInfo(param); } @Override public ConnectorResponse testConnect(TestConnectionRequestParam param) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(param.getDataSourceParam(), JdbcConnectionInfo.class); - BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(jdbcConnectionInfo); + Map paramMap = JSONUtils.toMap(param.getDataSourceParam()); + + BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(paramMap); dataSourceInfo.loadClass(); Properties properties = new Properties(); properties.setProperty(USER, dataSourceInfo.getUser()); @@ -101,4 +101,5 @@ public ConnectorResponse testConnect(TestConnectionRequestParam param) { public List keyProperties() { return Arrays.asList("host","port","catalog","database"); } + } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoDataSourceInfo.java index a4082b840..cf6fa4fdb 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class PrestoDataSourceInfo extends BaseJdbcDataSourceInfo { - public PrestoDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public PrestoDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoExecutor.java index 57904ea44..afbe2ac95 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class PrestoExecutor extends BaseJdbcExecutor { public PrestoExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public PrestoExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new PrestoDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new PrestoDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnector.java index 416ce31c3..915bcbc26 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnector.java @@ -17,13 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; public class SqlServerConnector extends JdbcConnector { @@ -32,8 +32,8 @@ public SqlServerConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new SqlServerDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new SqlServerDataSourceInfo(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDataSourceInfo.java index bc2488975..522bf34fb 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerDataSourceInfo.java @@ -17,12 +17,12 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; +import java.util.Map; public class SqlServerDataSourceInfo extends BaseJdbcDataSourceInfo { - public SqlServerDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public SqlServerDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerExecutor.java index be0d3e2f9..a887c8873 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerExecutor.java @@ -17,17 +17,18 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class SqlServerExecutor extends BaseJdbcExecutor { - public SqlServerExecutor(DataSourceClient DataSourceClient) { - super(DataSourceClient); + public SqlServerExecutor(DataSourceClient dataSourceClient) { + super(dataSourceClient); } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new SqlServerDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new SqlServerDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java index 20d446243..f53f8b923 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnector.java @@ -17,7 +17,6 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.datasource.jdbc.entity.ColumnInfo; import io.datavines.common.datasource.jdbc.entity.TableColumnInfo; import io.datavines.common.datasource.jdbc.entity.TableInfo; @@ -28,6 +27,7 @@ import io.datavines.common.utils.JSONUtils; import io.datavines.common.utils.StringUtils; import io.datavines.connector.api.DataSourceClient; +import org.apache.commons.collections4.MapUtils; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -35,6 +35,9 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import static io.datavines.common.ConfigConstants.CATALOG; public class StarRocksConnector extends MysqlConnector { @@ -43,8 +46,8 @@ public StarRocksConnector(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new StarRocksDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new StarRocksDataSourceInfo(param); } @Override @@ -52,12 +55,12 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); List tableList = null; ResultSet tables; @@ -97,20 +100,20 @@ public ConnectorResponse getTables(GetTablesRequestParam param) throws SQLExcept public ConnectorResponse getColumns(GetColumnsRequestParam param) throws SQLException { ConnectorResponse.ConnectorResponseBuilder builder = ConnectorResponse.builder(); String dataSourceParam = param.getDataSourceParam(); - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(dataSourceParam, JdbcConnectionInfo.class); - if (jdbcConnectionInfo == null) { + Map paramMap = JSONUtils.toMap(dataSourceParam); + if (MapUtils.isEmpty(paramMap)) { throw new SQLException("jdbc datasource param is no validate"); } - Connection connection = getConnection(dataSourceParam, jdbcConnectionInfo); + Connection connection = getConnection(dataSourceParam, paramMap); TableColumnInfo tableColumnInfo = null; try { String catalog; String schema; - if (StringUtils.isNotEmpty(jdbcConnectionInfo.getCatalog())) { - catalog = jdbcConnectionInfo.getCatalog(); + if (StringUtils.isNotEmpty(paramMap.get(CATALOG))) { + catalog = paramMap.get(CATALOG); schema = param.getDataBase(); } else { catalog = null; diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksDataSourceInfo.java index f5e0dba6a..5331c3097 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksDataSourceInfo.java @@ -16,13 +16,14 @@ */ package io.datavines.connector.plugin; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.utils.StringUtils; +import java.util.Map; + public class StarRocksDataSourceInfo extends MysqlDataSourceInfo { - public StarRocksDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public StarRocksDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksExecutor.java index d85ebd1c2..ccaf0067d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class StarRocksExecutor extends MysqlExecutor { public StarRocksExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public StarRocksExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new StarRocksDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new StarRocksDataSourceInfo(param); } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnector.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnector.java index afd839aa5..b21ddedea 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnector.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnector.java @@ -17,11 +17,8 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.common.param.ConnectorResponse; import io.datavines.common.param.TestConnectionRequestParam; -import io.datavines.common.param.form.Validate; -import io.datavines.common.param.form.type.InputParam; import io.datavines.common.utils.JSONUtils; import io.datavines.common.utils.StringUtils; import io.datavines.connector.api.DataSourceClient; @@ -29,6 +26,7 @@ import java.sql.*; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import static io.datavines.common.ConfigConstants.PASSWORD; @@ -52,20 +50,20 @@ public ResultSet getMetadataDatabases(Connection connection) throws SQLException } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new TrinoDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new TrinoDataSourceInfo(param); } @Override public ConnectorResponse testConnect(TestConnectionRequestParam param) { - JdbcConnectionInfo jdbcConnectionInfo = JSONUtils.parseObject(param.getDataSourceParam(), JdbcConnectionInfo.class); - BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(jdbcConnectionInfo); + Map paramMap = JSONUtils.toMap(param.getDataSourceParam()); + BaseJdbcDataSourceInfo dataSourceInfo = getDatasourceInfo(paramMap); dataSourceInfo.loadClass(); Properties properties = new Properties(); properties.setProperty(USER, dataSourceInfo.getUser()); if (StringUtils.isNotEmpty(dataSourceInfo.getPassword())) { - properties.setProperty(PASSWORD, dataSourceInfo.getUser()); + properties.setProperty(PASSWORD, dataSourceInfo.getPassword()); } String[] url2Array = dataSourceInfo.getJdbcUrl().split("\\?"); diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoDataSourceInfo.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoDataSourceInfo.java index caa39eb6e..f30f769de 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoDataSourceInfo.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoDataSourceInfo.java @@ -17,12 +17,13 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; + +import java.util.Map; public class TrinoDataSourceInfo extends BaseJdbcDataSourceInfo { - public TrinoDataSourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - super(jdbcConnectionInfo); + public TrinoDataSourceInfo(Map param) { + super(param); } @Override diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoExecutor.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoExecutor.java index fd0709253..9dd03da88 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoExecutor.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoExecutor.java @@ -17,9 +17,10 @@ package io.datavines.connector.plugin; import io.datavines.common.datasource.jdbc.BaseJdbcDataSourceInfo; -import io.datavines.common.datasource.jdbc.JdbcConnectionInfo; import io.datavines.connector.api.DataSourceClient; +import java.util.Map; + public class TrinoExecutor extends BaseJdbcExecutor { public TrinoExecutor(DataSourceClient dataSourceClient) { @@ -27,7 +28,7 @@ public TrinoExecutor(DataSourceClient dataSourceClient) { } @Override - public BaseJdbcDataSourceInfo getDatasourceInfo(JdbcConnectionInfo jdbcConnectionInfo) { - return new TrinoDataSourceInfo(jdbcConnectionInfo); + public BaseJdbcDataSourceInfo getDatasourceInfo(Map param) { + return new TrinoDataSourceInfo(param); } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java index 899f19a62..97a6211b6 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java @@ -18,6 +18,7 @@ import io.datavines.common.config.CheckResult; import io.datavines.common.config.Config; +import io.datavines.common.utils.ThreadUtils; import io.datavines.connector.api.ConnectorFactory; import io.datavines.connector.api.entity.JdbcOptions; import io.datavines.connector.plugin.utils.JdbcUtils; @@ -94,19 +95,23 @@ public ConnectionHolder getConnectionItem(LocalRuntimeEnvironment env) { @Override public boolean checkTableExist() { - if (connectionHolder != null) { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class) - .getOrCreatePlugin(config.getString(SRC_CONNECTOR_TYPE)); - JdbcOptions jdbcOptions = new JdbcOptions(); - jdbcOptions.setDatabaseName(config.getString(DATABASE)); - jdbcOptions.setSchemaName(config.getString(SCHEMA)); - jdbcOptions.setTableName(config.getString(TABLE)); - jdbcOptions.setQueryTimeout(10000); - try { - return JdbcUtils.tableExists(connectionHolder.getConnection(), jdbcOptions, connectorFactory.getDialect()); - } catch (Exception e) { - log.error("check table {} exists error :", config.getString(TABLE), e); - return false; + int retryTimes = 3; + while (retryTimes > 0) { + if (connectionHolder != null) { + ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class) + .getOrCreatePlugin(config.getString(SRC_CONNECTOR_TYPE)); + JdbcOptions jdbcOptions = new JdbcOptions(); + jdbcOptions.setDatabaseName(config.getString(DATABASE)); + jdbcOptions.setSchemaName(config.getString(SCHEMA)); + jdbcOptions.setTableName(config.getString(TABLE)); + jdbcOptions.setQueryTimeout(10000); + try { + return JdbcUtils.tableExists(connectionHolder.getConnection(), jdbcOptions, connectorFactory.getDialect()); + } catch (Exception e) { + log.error("check table {} exists error :", config.getString(TABLE), e); + retryTimes--; + ThreadUtils.sleep(2000); + } } } diff --git a/datavines-ui/src/App/index.tsx b/datavines-ui/src/App/index.tsx index 9922bae1e..083bd3f1f 100644 --- a/datavines-ui/src/App/index.tsx +++ b/datavines-ui/src/App/index.tsx @@ -12,7 +12,7 @@ import { routerNoLogin } from '../router'; import { IntlWrap } from '@/locale'; import { useMount } from '@/common'; import shareData from '@/utils/shareData'; -import { DV_LANGUAGE } from '@/utils/constants'; +import {DV_LANGUAGE, DV_STORAGE_LOGIN} from '@/utils/constants'; const Main = React.lazy(() => import('view/Main')); function App() { @@ -32,7 +32,7 @@ function App() {
- +