From 2f1b5ed0ba6a21f75c3b60c6ebad8f668e6a0fb5 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Mon, 18 Nov 2024 21:53:10 +0800 Subject: [PATCH 01/13] add jdbc support for Oracle --- externals/kyuubi-jdbc-engine/pom.xml | 12 ++ ...ine.jdbc.connection.JdbcConnectionProvider | 1 + ...che.kyuubi.engine.jdbc.dialect.JdbcDialect | 1 + .../jdbc/dialect/OracleSQLDialect.scala | 147 ++++++++++++++++++ .../oracle/OracleConnectionProvider.scala | 26 ++++ .../jdbc/oracle/OracleSchemaHelper.scala | 34 ++++ .../jdbc/oracle/OracleTRowSetGenerator.scala | 21 +++ .../engine/jdbc/schema/SchemaHelper.scala | 2 +- .../OperationWithOracleEngineSuite.scala | 61 ++++++++ .../jdbc/oracle/OracleOperationSuite.scala | 107 +++++++++++++ .../jdbc/oracle/OracleSessionSuite.scala | 40 +++++ .../jdbc/oracle/OracleStatementSuite.scala | 126 +++++++++++++++ .../jdbc/oracle/WithOracleContainer.scala | 35 +++++ .../engine/jdbc/oracle/WithOracleEngine.scala | 39 +++++ .../kyuubi/jdbc/hive/KyuubiBaseResultSet.java | 4 + pom.xml | 13 ++ 16 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala create mode 100644 externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleConnectionProvider.scala create mode 100644 externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSchemaHelper.scala create mode 100644 externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OperationWithOracleEngineSuite.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSessionSuite.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala create mode 100644 externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala diff --git a/externals/kyuubi-jdbc-engine/pom.xml b/externals/kyuubi-jdbc-engine/pom.xml index a69b55c4064..a32dbfbcc03 100644 --- a/externals/kyuubi-jdbc-engine/pom.xml +++ b/externals/kyuubi-jdbc-engine/pom.xml @@ -78,6 +78,12 @@ test + + com.dimafeng + testcontainers-scala-oracle-xe_${scala.binary.version} + test + + com.dimafeng testcontainers-scala-clickhouse_${scala.binary.version} @@ -115,6 +121,12 @@ http test + + + com.oracle.database.jdbc + ojdbc8 + test + diff --git a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider index 637de2e1e72..d7184dba4ba 100644 --- a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider +++ b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider @@ -19,6 +19,7 @@ org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseConnectionProvider org.apache.kyuubi.engine.jdbc.doris.DorisConnectionProvider org.apache.kyuubi.engine.jdbc.impala.ImpalaConnectionProvider org.apache.kyuubi.engine.jdbc.mysql.MySQLConnectionProvider +org.apache.kyuubi.engine.jdbc.oracle.OracleConnectionProvider org.apache.kyuubi.engine.jdbc.phoenix.PhoenixConnectionProvider org.apache.kyuubi.engine.jdbc.postgresql.PostgreSQLConnectionProvider org.apache.kyuubi.engine.jdbc.starrocks.StarRocksConnectionProvider diff --git a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect index 6c0838c0d02..ddd0383d3bd 100644 --- a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect +++ b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect @@ -19,6 +19,7 @@ org.apache.kyuubi.engine.jdbc.dialect.ClickHouseDialect org.apache.kyuubi.engine.jdbc.dialect.DorisDialect org.apache.kyuubi.engine.jdbc.dialect.ImpalaDialect org.apache.kyuubi.engine.jdbc.dialect.MySQLDialect +org.apache.kyuubi.engine.jdbc.dialect.OracleSQLDialect org.apache.kyuubi.engine.jdbc.dialect.PhoenixDialect org.apache.kyuubi.engine.jdbc.dialect.PostgreSQLDialect org.apache.kyuubi.engine.jdbc.dialect.StarRocksDialect diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala new file mode 100644 index 00000000000..4bab3d45d28 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.dialect + +import java.sql.{Connection, ResultSet, Statement} +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +import org.apache.commons.lang3.StringUtils + +import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.engine.jdbc.oracle.{OracleSchemaHelper, OracleTRowSetGenerator} +import org.apache.kyuubi.engine.jdbc.schema.{JdbcTRowSetGenerator, SchemaHelper} +import org.apache.kyuubi.operation.Operation +import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ +import org.apache.kyuubi.session.Session + +class OracleSQLDialect extends JdbcDialect { + + override def createStatement(connection: Connection, fetchSize: Int): Statement = { + val statement = + connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) + if (connection.getAutoCommit) { + statement.setFetchSize(fetchSize) + } + statement + } + + override def getTablesQuery( + catalog: String, + schema: String, + tableName: String, + tableTypes: util.List[String]): String = { + val tTypes = + if (tableTypes == null || tableTypes.isEmpty) { + Set() + } else { + tableTypes.asScala.toSet + } + val query = new StringBuilder( + s"""SELECT OWNER AS TABLE_SCHEMA, + | TABLE_NAME, + | TABLE_TYPE AS TABLE_TYPE + |FROM ALL_CATALOG + |""".stripMargin) + + val filters = ArrayBuffer[String]() + if (StringUtils.isNotBlank(schema)) { + filters += s"OWNER LIKE '$schema'" + } + + if (StringUtils.isNotBlank(tableName)) { + filters += s"$TABLE_NAME LIKE '$tableName'" + } + + if (tTypes.nonEmpty) { + filters += s"(${ + tTypes.map { tableType => s"$TABLE_TYPE = '$tableType'" } + .mkString(" OR ") + })" + } + + if (filters.nonEmpty) { + query.append(" WHERE ") + query.append(filters.mkString(" AND ")) + } + + query.toString() + } + + override def getTableTypesOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def getColumnsQuery( + session: Session, + catalogName: String, + schemaName: String, + tableName: String, + columnName: String): String = { + val query = new StringBuilder( + """ + |SELECT OWNER AS TABLE_SCHEMA + | , TABLE_NAME + | , COLUMN_NAME + |FROM ALL_TAB_COLUMNS + |""".stripMargin) + + val filters = ArrayBuffer[String]() + if (StringUtils.isNotEmpty(schemaName)) { + filters += s"OWNER LIKE '$schemaName'" + } + if (StringUtils.isNotEmpty(tableName)) { + filters += s"$TABLE_NAME LIKE '$tableName'" + } + if (StringUtils.isNotEmpty(columnName)) { + filters += s"$COLUMN_NAME LIKE '$columnName'" + } + + if (filters.nonEmpty) { + query.append(" WHERE ") + query.append(filters.mkString(" AND ")) + } + + query.toString() + } + + override def getFunctionsOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def getPrimaryKeysOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def getCrossReferenceOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def getTRowSetGenerator(): JdbcTRowSetGenerator = new OracleTRowSetGenerator + + override def getSchemaHelper(): SchemaHelper = { +// throw KyuubiSQLException.featureNotSupported() + new OracleSchemaHelper + } + + override def name(): String = { + "oracle" + } +} +// class OracleSchemaHelper extends SchemaHelper {} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleConnectionProvider.scala new file mode 100644 index 00000000000..11057193161 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleConnectionProvider.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.jdbc.oracle + +import org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider + +class OracleConnectionProvider extends JdbcConnectionProvider { + override val name: String = classOf[OracleConnectionProvider].getName + + override val driverClass: String = "oracle.jdbc.OracleDriver" +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSchemaHelper.scala new file mode 100644 index 00000000000..71783c8cd36 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSchemaHelper.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import java.sql.Types + +import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.TTypeDesc + +class OracleSchemaHelper extends SchemaHelper { + override protected def toTTypeDesc(sqlType: Int, precision: Int, scale: Int): TTypeDesc = { + sqlType match { + case Types.NUMERIC if scale == 0 => + super.toTTypeDesc(Types.INTEGER, precision, scale) + case Types.NUMERIC => + super.toTTypeDesc(Types.DECIMAL, precision, scale) + case _ => super.toTTypeDesc(sqlType, precision, scale) + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala new file mode 100644 index 00000000000..33f6b16f8d8 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import org.apache.kyuubi.engine.jdbc.schema.DefaultJdbcTRowSetGenerator + +class OracleTRowSetGenerator extends DefaultJdbcTRowSetGenerator {} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala index 16d46fc36f6..b6ea4035234 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala @@ -41,7 +41,7 @@ abstract class SchemaHelper { tColumnDesc } - private def toTTypeDesc(sqlType: Int, precision: Int, scale: Int): TTypeDesc = { + protected def toTTypeDesc(sqlType: Int, precision: Int, scale: Int): TTypeDesc = { val typeEntry = new TPrimitiveTypeEntry(toTTypeId(sqlType)) typeEntry.setTypeQualifiers(toTTypeQualifiers(sqlType, precision, scale)) val tTypeDesc = new TTypeDesc() diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OperationWithOracleEngineSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OperationWithOracleEngineSuite.scala new file mode 100644 index 00000000000..fef37558091 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OperationWithOracleEngineSuite.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.jdbc.connection.ConnectionProvider +import org.apache.kyuubi.operation.HiveJDBCTestHelper +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.{TGetInfoReq, TGetInfoType} + +class OperationWithOracleEngineSuite extends OracleOperationSuite with HiveJDBCTestHelper { + + override protected def jdbcUrl: String = jdbcConnectionUrl + + test("oracle - test for Jdbc engine getInfo") { + val metaData = ConnectionProvider.create(kyuubiConf).getMetaData + + withSessionConf(Map(KyuubiConf.SERVER_INFO_PROVIDER.key -> "ENGINE"))()() { + withSessionHandle { (client, handle) => + val req = new TGetInfoReq() + req.setSessionHandle(handle) + req.setInfoType(TGetInfoType.CLI_DBMS_NAME) + assert(client.GetInfo(req).getInfoValue.getStringValue == metaData.getDatabaseProductName) + + val req2 = new TGetInfoReq() + req2.setSessionHandle(handle) + req2.setInfoType(TGetInfoType.CLI_DBMS_VER) + assert( + client.GetInfo(req2).getInfoValue.getStringValue == metaData.getDatabaseProductVersion) + + val req3 = new TGetInfoReq() + req3.setSessionHandle(handle) + req3.setInfoType(TGetInfoType.CLI_MAX_COLUMN_NAME_LEN) + assert(client.GetInfo(req3).getInfoValue.getLenValue == metaData.getMaxColumnNameLength) + + val req4 = new TGetInfoReq() + req4.setSessionHandle(handle) + req4.setInfoType(TGetInfoType.CLI_MAX_SCHEMA_NAME_LEN) + assert(client.GetInfo(req4).getInfoValue.getLenValue == metaData.getMaxSchemaNameLength) + + val req5 = new TGetInfoReq() + req5.setSessionHandle(handle) + req5.setInfoType(TGetInfoType.CLI_MAX_TABLE_NAME_LEN) + assert(client.GetInfo(req5).getInfoValue.getLenValue == metaData.getMaxTableNameLength) + } + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala new file mode 100644 index 00000000000..66a62195906 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.operation.HiveJDBCTestHelper +import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.{COLUMN_NAME, TABLE_NAME, TABLE_TYPE} + +abstract class OracleOperationSuite extends WithOracleEngine with HiveJDBCTestHelper { + test("oracle - get tables") { + case class Table(catalog: String, schema: String, tableName: String, tableType: String) + + withJdbcStatement() { statement => + val meta = statement.getConnection.getMetaData + val resultBuffer = ArrayBuffer[Table]() + + var tables = meta.getTables(null, null, null, null) + while (tables.next()) { + resultBuffer += + Table( + null, + null, + tables.getString(TABLE_NAME), + tables.getString(TABLE_TYPE)) + } + assert(resultBuffer.contains(Table(null, null, "DUAL", "TABLE"))) + assert(resultBuffer.contains(Table(null, null, "DUAL", "SYNONYM"))) + assert(resultBuffer.contains(Table(null, null, "NLS_SESSION_PARAMETERS", "VIEW"))) + resultBuffer.clear() + + statement.execute( + """create table T_PEOPLE + |(ID INTEGER not null constraint "T_PEOPLE_pk" primary key, + |NAME VARCHAR2(64))""".stripMargin) + + tables = meta.getTables(null, null, "T_PEOPLE", Array("TABLE")) + while (tables.next()) { + val table = Table( + null, + null, + tables.getString(TABLE_NAME), + tables.getString(TABLE_TYPE)) + assert(table == Table(null, null, "T_PEOPLE", "TABLE")) + } + + tables = meta.getTables(null, null, "%PEOPLE", Array("TABLE")) + while (tables.next()) { + val table = Table( + null, + null, + tables.getString(TABLE_NAME), + tables.getString(TABLE_TYPE)) + assert(table == Table(null, null, "T_PEOPLE", "TABLE")) + } + + statement.execute("DROP TABLE T_PEOPLE") + } + } + + test("oracle - get columns") { + case class Column(tableName: String, columnName: String) + + withJdbcStatement() { statement => + val meta = statement.getConnection.getMetaData + val resultBuffer = ArrayBuffer[Column]() + + var columns = meta.getColumns(null, null, null, null) + while (columns.next()) { + resultBuffer += + Column( + columns.getString(TABLE_NAME), + columns.getString(COLUMN_NAME)) + } + assert(resultBuffer.contains(Column("DUAL", "DUMMY"))) + resultBuffer.clear() + + statement.execute( + """create table T_PEOPLE + |(ID INTEGER not null constraint "T_PEOPLE_pk" primary key)""".stripMargin) + + columns = meta.getColumns(null, null, "%PEOPLE", null) + while (columns.next()) { + val column = Column( + columns.getString(TABLE_NAME), + columns.getString(COLUMN_NAME)) + assert(column == Column("T_PEOPLE", "ID")) + } + + statement.execute("DROP TABLE T_PEOPLE") + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSessionSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSessionSuite.scala new file mode 100644 index 00000000000..9a9843617b5 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleSessionSuite.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +class OracleSessionSuite extends WithOracleEngine with HiveJDBCTestHelper { + test("oracle session suite") { + withJdbcStatement() { statement => + { + val resultSet = statement.executeQuery( + "SELECT '1' AS ID FROM DUAL") + val metadata = resultSet.getMetaData + for (i <- 1 to metadata.getColumnCount) { + assert(metadata.getColumnName(i) == "ID") + } + while (resultSet.next()) { + val id = resultSet.getObject(1) + assert(id == "1") + } + } + } + } + + override protected def jdbcUrl: String = jdbcConnectionUrl +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala new file mode 100644 index 00000000000..3869b206af3 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.oracle + +import java.sql.Timestamp + +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +class OracleStatementSuite extends WithOracleEngine with HiveJDBCTestHelper { + + test("oracle - test select") { + withJdbcStatement() { statement => + statement.execute( + """create table T_TEST + |( + | ID INTEGER not null + | constraint "T_TEST_PK" primary key, + | NAME VARCHAR2(64), + | USER_ID LONG, + | SCORE NUMBER(8,2) + |)""".stripMargin) + statement.execute("""INSERT INTO T_TEST(ID, NAME, USER_ID, SCORE) + |VALUES (1, 'Bob', 43254353,89.92)""".stripMargin) + val resultSet = statement.executeQuery("SELECT * FROM T_TEST") + while (resultSet.next()) { + val id = resultSet.getObject(1) + assert(id == 1) + val name = resultSet.getObject(2) + assert(name == "Bob") + val user_id = resultSet.getObject(3) + assert(user_id == "43254353") + val score = resultSet.getObject(4) + assert(score == new java.math.BigDecimal("89.92")) + } + statement.execute("DROP TABLE T_TEST") + } + } + + test("oracle - test types") { + withJdbcStatement() { statement => + statement.execute( + """ + |CREATE TABLE TYPE_TEST + |( + | INT_COL INTEGER, + | NUM_8_COL NUMBER(8, 0), + | NUM_16_4_COL NUMBER(16, 4), + | DECIMAL_COL DECIMAL(8, 2), + | DATE_COL DATE, + | TIMESTAMP_COL TIMESTAMP, + | CHAR_COL CHAR(10), + | VARCHAR2_COL VARCHAR2(255), + | NCHAR_COL NCHAR(10), + | NCHAR2_COL NVARCHAR2(255), + | LONG_COL LONG, + | FLOAT_COL FLOAT, + | REAL_COL REAL + |) + |""".stripMargin) + statement.execute( + """ + |INSERT INTO TYPE_TEST( INT_COL + | , NUM_8_COL + | , NUM_16_4_COL + | , DECIMAL_COL + | , DATE_COL + | , TIMESTAMP_COL + | , CHAR_COL + | , VARCHAR2_COL + | , NCHAR_COL + | , NCHAR2_COL + | , LONG_COL + | , FLOAT_COL + | , REAL_COL) + |VALUES ( 1 + | , 2 + | , 3.1415 + | , 0.61 + | , TO_DATE('2024-11-07', 'YYYY-MM-DD') + | , TO_TIMESTAMP('2024-11-07 22:03:01.324', 'YYYY-MM-DD HH24:MI:SS.FF3') + | , 'pi' + | , 'alice' + | , 'bob' + | , 'siri' + | , 'alex' + | , 1.432 + | , 3.432 + | ) + |""".stripMargin) + + val resultSet1 = statement.executeQuery("SELECT * FROM TYPE_TEST") + while (resultSet1.next()) { + assert(resultSet1.getObject(1) == 1) + assert(resultSet1.getObject(2) == 2) + assert(resultSet1.getObject(3) == new java.math.BigDecimal("3.1415")) + assert(resultSet1.getObject(4) == new java.math.BigDecimal("0.61")) + assert(resultSet1.getObject(5) == Timestamp.valueOf("2024-11-07 00:00:00")) + assert(resultSet1.getObject(6) == Timestamp.valueOf("2024-11-07 22:03:01.324")) + assert(resultSet1.getObject(7) == "pi ") + assert(resultSet1.getObject(8) == "alice") + assert(resultSet1.getObject(9) == "bob ") + assert(resultSet1.getObject(10) == "siri") + assert(resultSet1.getObject(11) == "alex") + assert(resultSet1.getObject(12) == new java.math.BigDecimal("1.432")) + assert(resultSet1.getObject(13) == new java.math.BigDecimal("3.432")) + } + statement.execute("DROP TABLE TYPE_TEST") + } + } + + override protected def jdbcUrl: String = jdbcConnectionUrl +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala new file mode 100644 index 00000000000..80f4313e55c --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.jdbc.oracle + +import com.dimafeng.testcontainers.OracleContainer +import org.testcontainers.utility.DockerImageName + +import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer + +trait WithOracleContainer extends WithJdbcServerContainer { + + private val oracleDockerImage = "gvenzl/oracle-xe:18.4.0-slim" + protected val oracleUserName = "kyuubi" + protected val oraclePassword = "oracle" + + override val containerDef = OracleContainer.Def( + dockerImageName = DockerImageName.parse(oracleDockerImage), + username = oracleUserName, + password = oraclePassword) +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala new file mode 100644 index 00000000000..71498078eb5 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.jdbc.oracle + +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_CONNECTION_USER, ENGINE_JDBC_DRIVER_CLASS, ENGINE_JDBC_SHORT_NAME, ENGINE_SHARE_LEVEL, ENGINE_TYPE, FRONTEND_BIND_HOST} +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY +import org.apache.kyuubi.engine.jdbc.WithJdbcEngine + +trait WithOracleEngine extends WithJdbcEngine with WithOracleContainer { + + override def withKyuubiConf: Map[String, String] = withContainers { container => + Map( + ENGINE_SHARE_LEVEL.key -> "SERVER", + ENGINE_JDBC_CONNECTION_URL.key -> container.jdbcUrl, + ENGINE_JDBC_CONNECTION_USER.key -> container.username, + ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, + ENGINE_TYPE.key -> "jdbc", + ENGINE_JDBC_SHORT_NAME.key -> "oracle", + FRONTEND_BIND_HOST.key -> "0.0.0.0", + KYUUBI_SESSION_USER_KEY -> "kyuubi", + ENGINE_JDBC_DRIVER_CLASS.key -> "oracle.jdbc.OracleDriver") + } + +} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java index cf47e104295..6d08236c115 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java @@ -338,6 +338,10 @@ private Object evaluate(TTypeId columnType, Object value) { if (value instanceof String) { return ((String) value).getBytes(); } + case INT_TYPE: + if (value instanceof String) { + return Integer.valueOf((String) value); + } return value; case TIMESTAMP_TYPE: return Timestamp.valueOf((String) value); diff --git a/pom.xml b/pom.xml index 1fa2e0b99b3..ac9ec6694f5 100644 --- a/pom.xml +++ b/pom.xml @@ -187,6 +187,7 @@ paimon-spark-${spark.binary.version} 6.0.0 42.7.2 + 23.2.0.0 0.16.0 3.25.5 3.2.16 @@ -583,6 +584,12 @@ ${testcontainers-scala.version} + + com.dimafeng + testcontainers-scala-oracle-xe_${scala.binary.version} + ${testcontainers-scala.version} + + com.dimafeng testcontainers-scala-clickhouse_${scala.binary.version} @@ -1091,6 +1098,12 @@ ${postgresql.version} + + com.oracle.database.jdbc + ojdbc8 + ${ojdbc.version} + + com.clickhouse clickhouse-jdbc From b43cea4214cce2f7ddd2b64b98a0fd4e58afa67e Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 19 Nov 2024 23:54:12 +0800 Subject: [PATCH 02/13] add oracle configuration for kyuubi.engine.jdbc.connection.provider --- docs/configuration/settings.md | 2 +- .../main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index a1b714d0d77..7485e076387 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -163,7 +163,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.engine.jdbc.connection.password | <undefined> | The password is used for connecting to server | string | 1.6.0 | | kyuubi.engine.jdbc.connection.propagateCredential | false | Whether to use the session's user and password to connect to database | boolean | 1.8.0 | | kyuubi.engine.jdbc.connection.properties || The additional properties are used for connecting to server | seq | 1.6.0 | -| kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
    • | string | 1.6.0 | +| kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
  • oracle: For establishing oracle connections.
    • | string | 1.6.0 | | kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | | kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | | kyuubi.engine.jdbc.deploy.mode | LOCAL | Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. In local mode, the engine operates on the same node as the KyuubiServer. In YARN mode, the engine runs within the Application Master (AM) container of YARN. | string | 1.10.0 | diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 436ac5fd16b..997915c77d9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3245,7 +3245,9 @@ object KyuubiConf { "
  • postgresql: For establishing PostgreSQL connections.
    • " + "
  • starrocks: For establishing StarRocks connections.
    • " + "
  • impala: For establishing Impala connections.
    • " + - "
  • clickhouse: For establishing clickhouse connections.
    • ") + "
  • clickhouse: For establishing clickhouse connections.
    • " + + "
  • oracle: For establishing oracle connections.
    • " + ) .version("1.6.0") .stringConf .transform { @@ -3263,6 +3265,8 @@ object KyuubiConf { "org.apache.kyuubi.engine.jdbc.impala.ImpalaConnectionProvider" case "ClickHouse" | "clickhouse" | "ClickHouseConnectionProvider" => "org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseConnectionProvider" + case "Oracle" | "oracle" | "OracleConnectionProvider" => + "org.apache.kyuubi.engine.jdbc.oracle.OracleConnectionProvider" case other => other } .createOptional From aa2e2e9ba370579f4105fe74c7f0e7dc5950962a Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 19 Nov 2024 23:56:03 +0800 Subject: [PATCH 03/13] adjust wired space in OracleSQLDialect --- .../apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index 4bab3d45d28..89f46d4459e 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -54,7 +54,7 @@ class OracleSQLDialect extends JdbcDialect { tableTypes.asScala.toSet } val query = new StringBuilder( - s"""SELECT OWNER AS TABLE_SCHEMA, + s"""SELECT OWNER AS TABLE_SCHEMA, | TABLE_NAME, | TABLE_TYPE AS TABLE_TYPE |FROM ALL_CATALOG From d1e82edb1e54597f83c3581b01564b9971feb5ed Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Wed, 20 Nov 2024 15:15:15 +0800 Subject: [PATCH 04/13] fix code style checking error in settings.md --- docs/configuration/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 7485e076387..fede369a842 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -163,7 +163,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.engine.jdbc.connection.password | <undefined> | The password is used for connecting to server | string | 1.6.0 | | kyuubi.engine.jdbc.connection.propagateCredential | false | Whether to use the session's user and password to connect to database | boolean | 1.8.0 | | kyuubi.engine.jdbc.connection.properties || The additional properties are used for connecting to server | seq | 1.6.0 | -| kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
  • oracle: For establishing oracle connections.
    • | string | 1.6.0 | +| kyuubi.engine.jdbc.connection.provider | <undefined> | A JDBC connection provider plugin for the Kyuubi Server to establish a connection to the JDBC URL. The configuration value should be a subclass of `org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider`. Kyuubi provides the following built-in implementations:
  • doris: For establishing Doris connections.
  • mysql: For establishing MySQL connections.
  • phoenix: For establishing Phoenix connections.
  • postgresql: For establishing PostgreSQL connections.
  • starrocks: For establishing StarRocks connections.
  • impala: For establishing Impala connections.
  • clickhouse: For establishing clickhouse connections.
  • oracle: For establishing oracle connections.
    • | string | 1.6.0 | | kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | | kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | | kyuubi.engine.jdbc.deploy.mode | LOCAL | Configures the jdbc engine deploy mode, The value can be 'local', 'yarn'. In local mode, the engine operates on the same node as the KyuubiServer. In YARN mode, the engine runs within the Application Master (AM) container of YARN. | string | 1.10.0 | From abf983727eeb094ad00f94ed30fd2bcc64b257a9 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Wed, 20 Nov 2024 21:49:48 +0800 Subject: [PATCH 05/13] fix code style checking error in KyuubiConf.scala --- .../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 997915c77d9..e26033bf0f8 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3246,8 +3246,7 @@ object KyuubiConf { "
  • starrocks: For establishing StarRocks connections.
    • " + "
  • impala: For establishing Impala connections.
    • " + "
  • clickhouse: For establishing clickhouse connections.
    • " + - "
  • oracle: For establishing oracle connections.
    • " - ) + "
  • oracle: For establishing oracle connections.
    • ") .version("1.6.0") .stringConf .transform { @@ -3266,7 +3265,7 @@ object KyuubiConf { case "ClickHouse" | "clickhouse" | "ClickHouseConnectionProvider" => "org.apache.kyuubi.engine.jdbc.clickhouse.ClickHouseConnectionProvider" case "Oracle" | "oracle" | "OracleConnectionProvider" => - "org.apache.kyuubi.engine.jdbc.oracle.OracleConnectionProvider" + "org.apache.kyuubi.engine.jdbc.oracle.OracleConnectionProvider" case other => other } .createOptional From d688b47061649f3023eb1990f2f6f70b6769c2e0 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Mon, 25 Nov 2024 23:38:37 +0800 Subject: [PATCH 06/13] change oracle image into gvenzl/oracle-free:23.5-slim --- .../jdbc/oracle/WithOracleContainer.scala | 36 ++++++++++++++----- .../engine/jdbc/oracle/WithOracleEngine.scala | 6 ++-- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala index 80f4313e55c..ed5217cef8d 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala @@ -17,19 +17,39 @@ package org.apache.kyuubi.engine.jdbc.oracle -import com.dimafeng.testcontainers.OracleContainer -import org.testcontainers.utility.DockerImageName +// import java.time.Duration + +import com.dimafeng.testcontainers.GenericContainer +import org.testcontainers.containers.wait.strategy.{Wait, WaitAllStrategy} import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer trait WithOracleContainer extends WithJdbcServerContainer { - - private val oracleDockerImage = "gvenzl/oracle-xe:18.4.0-slim" + private val oraclePort = 1521 + private val ports = Seq(oraclePort) + private val oracleDockerImage = "gvenzl/oracle-free:23.5-slim" + private val oracleUsernameKey = "APP_USER" + private val oraclePasswordKey = "APP_USER_PASSWORD" protected val oracleUserName = "kyuubi" protected val oraclePassword = "oracle" - override val containerDef = OracleContainer.Def( - dockerImageName = DockerImageName.parse(oracleDockerImage), - username = oracleUserName, - password = oraclePassword) + override val containerDef: GenericContainer.Def[GenericContainer] = GenericContainer.Def( + dockerImage = oracleDockerImage, + exposedPorts = ports, + waitStrategy = new WaitAllStrategy() + // .withStartupTimeout(Duration.ofMinutes(1)) + .withStrategy(Wait.forListeningPort()) + .withStrategy(Wait.forLogMessage( + "Completed: Pluggable database FREEPDB1 opened read write", 1)) + , + env = Map( + oraclePasswordKey -> oraclePassword, + oracleUsernameKey -> oracleUserName, + "ORACLE_RANDOM_PASSWORD" -> "true")) + + protected def oracleJdbcUrl: String = withContainers { container => + val queryServerHost: String = container.host + val queryServerPort: Int = container.mappedPort(oraclePort) + s"jdbc:oracle:thin:@//$queryServerHost:$queryServerPort/FREEPDB1" + } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala index 71498078eb5..d726028a31d 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala @@ -26,9 +26,9 @@ trait WithOracleEngine extends WithJdbcEngine with WithOracleContainer { override def withKyuubiConf: Map[String, String] = withContainers { container => Map( ENGINE_SHARE_LEVEL.key -> "SERVER", - ENGINE_JDBC_CONNECTION_URL.key -> container.jdbcUrl, - ENGINE_JDBC_CONNECTION_USER.key -> container.username, - ENGINE_JDBC_CONNECTION_PASSWORD.key -> container.password, + ENGINE_JDBC_CONNECTION_URL.key -> oracleJdbcUrl, + ENGINE_JDBC_CONNECTION_USER.key -> oracleUserName, + ENGINE_JDBC_CONNECTION_PASSWORD.key -> oraclePassword, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "oracle", FRONTEND_BIND_HOST.key -> "0.0.0.0", From 4c967b98e014ac6e3529e78a26cde3b3dc7293ef Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:03:42 +0800 Subject: [PATCH 07/13] use gvenzl/oracle-free:23.5-slim with docker-compose for test case --- .../src/test/resources/oracle-compose.yml | 42 ++++++++++++++++ .../jdbc/oracle/WithOracleContainer.scala | 49 +++++++++---------- .../engine/jdbc/oracle/WithOracleEngine.scala | 7 ++- 3 files changed, 68 insertions(+), 30 deletions(-) create mode 100644 externals/kyuubi-jdbc-engine/src/test/resources/oracle-compose.yml diff --git a/externals/kyuubi-jdbc-engine/src/test/resources/oracle-compose.yml b/externals/kyuubi-jdbc-engine/src/test/resources/oracle-compose.yml new file mode 100644 index 00000000000..17fd3c3c82a --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/resources/oracle-compose.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3" +services: + + # Oracle service (label used to access the service container) + oracle: + + # Docker Hub image (feel free to change the tag "latest" to any other available one) + image: gvenzl/oracle-free:23.5-slim + + # Provide passwords and other environment variables to container + environment: + ORACLE_RANDOM_PASSWORD: true + APP_USER: kyuubi + APP_USER_PASSWORD: oracle + + # Forward Oracle port + ports: + - "1521" + + # Provide healthcheck script options for startup + healthcheck: + test: healthcheck.sh + interval: 10s + timeout: 5s + retries: 10 \ No newline at end of file diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala index ed5217cef8d..5e69b98fd5c 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleContainer.scala @@ -17,39 +17,36 @@ package org.apache.kyuubi.engine.jdbc.oracle -// import java.time.Duration +import java.io.File +import java.time.Duration -import com.dimafeng.testcontainers.GenericContainer -import org.testcontainers.containers.wait.strategy.{Wait, WaitAllStrategy} +import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService} +import org.testcontainers.containers.wait.strategy.DockerHealthcheckWaitStrategy +import org.apache.kyuubi.Utils import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer trait WithOracleContainer extends WithJdbcServerContainer { - private val oraclePort = 1521 - private val ports = Seq(oraclePort) - private val oracleDockerImage = "gvenzl/oracle-free:23.5-slim" - private val oracleUsernameKey = "APP_USER" - private val oraclePasswordKey = "APP_USER_PASSWORD" - protected val oracleUserName = "kyuubi" - protected val oraclePassword = "oracle" + private val ORACLE_PORT = 1521 + private val ORACLE_SERVICE_NAME = "oracle" + protected val ORACLE_USER_NAME = "kyuubi" + protected val ORACLE_PASSWORD = "oracle" - override val containerDef: GenericContainer.Def[GenericContainer] = GenericContainer.Def( - dockerImage = oracleDockerImage, - exposedPorts = ports, - waitStrategy = new WaitAllStrategy() - // .withStartupTimeout(Duration.ofMinutes(1)) - .withStrategy(Wait.forListeningPort()) - .withStrategy(Wait.forLogMessage( - "Completed: Pluggable database FREEPDB1 opened read write", 1)) - , - env = Map( - oraclePasswordKey -> oraclePassword, - oracleUsernameKey -> oracleUserName, - "ORACLE_RANDOM_PASSWORD" -> "true")) + override val containerDef: DockerComposeContainer.Def = + DockerComposeContainer + .Def( + composeFiles = new File(Utils.getContextOrKyuubiClassLoader + .getResource("oracle-compose.yml").toURI), + exposedServices = Seq[ExposedService]( + ExposedService( + ORACLE_SERVICE_NAME, + ORACLE_PORT, + waitStrategy = + new DockerHealthcheckWaitStrategy().withStartupTimeout(Duration.ofMinutes(2))))) protected def oracleJdbcUrl: String = withContainers { container => - val queryServerHost: String = container.host - val queryServerPort: Int = container.mappedPort(oraclePort) - s"jdbc:oracle:thin:@//$queryServerHost:$queryServerPort/FREEPDB1" + val feHost: String = container.getServiceHost(ORACLE_SERVICE_NAME, ORACLE_PORT) + val fePort: Int = container.getServicePort(ORACLE_SERVICE_NAME, ORACLE_PORT) + s"jdbc:oracle:thin:@//$feHost:$fePort/FREEPDB1" } } diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala index d726028a31d..f30e6d1951c 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/WithOracleEngine.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.engine.jdbc.oracle -import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_CONNECTION_USER, ENGINE_JDBC_DRIVER_CLASS, ENGINE_JDBC_SHORT_NAME, ENGINE_SHARE_LEVEL, ENGINE_TYPE, FRONTEND_BIND_HOST} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_CONNECTION_USER, ENGINE_JDBC_DRIVER_CLASS, ENGINE_JDBC_SHORT_NAME, ENGINE_SHARE_LEVEL, ENGINE_TYPE} import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.jdbc.WithJdbcEngine @@ -27,11 +27,10 @@ trait WithOracleEngine extends WithJdbcEngine with WithOracleContainer { Map( ENGINE_SHARE_LEVEL.key -> "SERVER", ENGINE_JDBC_CONNECTION_URL.key -> oracleJdbcUrl, - ENGINE_JDBC_CONNECTION_USER.key -> oracleUserName, - ENGINE_JDBC_CONNECTION_PASSWORD.key -> oraclePassword, + ENGINE_JDBC_CONNECTION_USER.key -> ORACLE_USER_NAME, + ENGINE_JDBC_CONNECTION_PASSWORD.key -> ORACLE_PASSWORD, ENGINE_TYPE.key -> "jdbc", ENGINE_JDBC_SHORT_NAME.key -> "oracle", - FRONTEND_BIND_HOST.key -> "0.0.0.0", KYUUBI_SESSION_USER_KEY -> "kyuubi", ENGINE_JDBC_DRIVER_CLASS.key -> "oracle.jdbc.OracleDriver") } From b33e97a08afeb3a563e51914901ba0d0f29fa03d Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:06:04 +0800 Subject: [PATCH 08/13] remove redundant testcontainers-scala-oracle-xe dependency in pom.xml --- externals/kyuubi-jdbc-engine/pom.xml | 6 ------ pom.xml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/pom.xml b/externals/kyuubi-jdbc-engine/pom.xml index a32dbfbcc03..4a3ee709344 100644 --- a/externals/kyuubi-jdbc-engine/pom.xml +++ b/externals/kyuubi-jdbc-engine/pom.xml @@ -78,12 +78,6 @@ test     - - com.dimafeng - testcontainers-scala-oracle-xe_${scala.binary.version} - test - - com.dimafeng testcontainers-scala-clickhouse_${scala.binary.version} diff --git a/pom.xml b/pom.xml index c86bd6fe11e..2bc1d2ce439 100644 --- a/pom.xml +++ b/pom.xml @@ -584,12 +584,6 @@ ${testcontainers-scala.version} - - com.dimafeng - testcontainers-scala-oracle-xe_${scala.binary.version} - ${testcontainers-scala.version} - - com.dimafeng testcontainers-scala-clickhouse_${scala.binary.version} From e5834846013539b8e6c53c0f676e0ea9b47e4879 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:10:14 +0800 Subject: [PATCH 09/13] Update externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala Co-authored-by: Cheng Pan --- .../apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index 89f46d4459e..b419127213e 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -140,8 +140,6 @@ class OracleSQLDialect extends JdbcDialect { new OracleSchemaHelper } - override def name(): String = { - "oracle" - } + override def name(): String = "oracle" } // class OracleSchemaHelper extends SchemaHelper {} From 70b49fcba2a936d03150bcb83aa12a08f5426bd9 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:41:00 +0800 Subject: [PATCH 10/13] Use the single line string if SQL fits in one line, otherwise write it in a pretty style --- .../jdbc/dialect/OracleSQLDialect.scala | 15 +--- .../jdbc/oracle/OracleOperationSuite.scala | 11 ++- .../jdbc/oracle/OracleStatementSuite.scala | 71 ++++--------------- 3 files changed, 19 insertions(+), 78 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index b419127213e..4394170c7c3 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -54,11 +54,7 @@ class OracleSQLDialect extends JdbcDialect { tableTypes.asScala.toSet } val query = new StringBuilder( - s"""SELECT OWNER AS TABLE_SCHEMA, - | TABLE_NAME, - | TABLE_TYPE AS TABLE_TYPE - |FROM ALL_CATALOG - |""".stripMargin) + "SELECT OWNER AS TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE AS TABLE_TYPE FROM ALL_CATALOG") val filters = ArrayBuffer[String]() if (StringUtils.isNotBlank(schema)) { @@ -95,12 +91,7 @@ class OracleSQLDialect extends JdbcDialect { tableName: String, columnName: String): String = { val query = new StringBuilder( - """ - |SELECT OWNER AS TABLE_SCHEMA - | , TABLE_NAME - | , COLUMN_NAME - |FROM ALL_TAB_COLUMNS - |""".stripMargin) + "SELECT OWNER AS TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME FROM ALL_TAB_COLUMNS") val filters = ArrayBuffer[String]() if (StringUtils.isNotEmpty(schemaName)) { @@ -136,10 +127,8 @@ class OracleSQLDialect extends JdbcDialect { override def getTRowSetGenerator(): JdbcTRowSetGenerator = new OracleTRowSetGenerator override def getSchemaHelper(): SchemaHelper = { -// throw KyuubiSQLException.featureNotSupported() new OracleSchemaHelper } override def name(): String = "oracle" } -// class OracleSchemaHelper extends SchemaHelper {} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala index 66a62195906..1cec7e2cd17 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleOperationSuite.scala @@ -43,10 +43,8 @@ abstract class OracleOperationSuite extends WithOracleEngine with HiveJDBCTestHe assert(resultBuffer.contains(Table(null, null, "NLS_SESSION_PARAMETERS", "VIEW"))) resultBuffer.clear() - statement.execute( - """create table T_PEOPLE - |(ID INTEGER not null constraint "T_PEOPLE_pk" primary key, - |NAME VARCHAR2(64))""".stripMargin) + statement.execute("create table T_PEOPLE (ID INTEGER not null " + + "constraint \"T_PEOPLE_pk\" primary key, NAME VARCHAR2(64))") tables = meta.getTables(null, null, "T_PEOPLE", Array("TABLE")) while (tables.next()) { @@ -89,9 +87,8 @@ abstract class OracleOperationSuite extends WithOracleEngine with HiveJDBCTestHe assert(resultBuffer.contains(Column("DUAL", "DUMMY"))) resultBuffer.clear() - statement.execute( - """create table T_PEOPLE - |(ID INTEGER not null constraint "T_PEOPLE_pk" primary key)""".stripMargin) + statement.execute("create table T_PEOPLE (ID INTEGER not null " + + "constraint \"T_PEOPLE_pk\" primary key)") columns = meta.getColumns(null, null, "%PEOPLE", null) while (columns.next()) { diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala index 3869b206af3..91e9b0ad835 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala @@ -24,15 +24,8 @@ class OracleStatementSuite extends WithOracleEngine with HiveJDBCTestHelper { test("oracle - test select") { withJdbcStatement() { statement => - statement.execute( - """create table T_TEST - |( - | ID INTEGER not null - | constraint "T_TEST_PK" primary key, - | NAME VARCHAR2(64), - | USER_ID LONG, - | SCORE NUMBER(8,2) - |)""".stripMargin) + statement.execute("create table T_TEST (ID INTEGER not null " + + "constraint \"T_TEST_PK\" primary key, NAME VARCHAR2(64), USER_ID LONG, SCORE NUMBER(8,2))") statement.execute("""INSERT INTO T_TEST(ID, NAME, USER_ID, SCORE) |VALUES (1, 'Bob', 43254353,89.92)""".stripMargin) val resultSet = statement.executeQuery("SELECT * FROM T_TEST") @@ -52,55 +45,17 @@ class OracleStatementSuite extends WithOracleEngine with HiveJDBCTestHelper { test("oracle - test types") { withJdbcStatement() { statement => - statement.execute( - """ - |CREATE TABLE TYPE_TEST - |( - | INT_COL INTEGER, - | NUM_8_COL NUMBER(8, 0), - | NUM_16_4_COL NUMBER(16, 4), - | DECIMAL_COL DECIMAL(8, 2), - | DATE_COL DATE, - | TIMESTAMP_COL TIMESTAMP, - | CHAR_COL CHAR(10), - | VARCHAR2_COL VARCHAR2(255), - | NCHAR_COL NCHAR(10), - | NCHAR2_COL NVARCHAR2(255), - | LONG_COL LONG, - | FLOAT_COL FLOAT, - | REAL_COL REAL - |) - |""".stripMargin) - statement.execute( - """ - |INSERT INTO TYPE_TEST( INT_COL - | , NUM_8_COL - | , NUM_16_4_COL - | , DECIMAL_COL - | , DATE_COL - | , TIMESTAMP_COL - | , CHAR_COL - | , VARCHAR2_COL - | , NCHAR_COL - | , NCHAR2_COL - | , LONG_COL - | , FLOAT_COL - | , REAL_COL) - |VALUES ( 1 - | , 2 - | , 3.1415 - | , 0.61 - | , TO_DATE('2024-11-07', 'YYYY-MM-DD') - | , TO_TIMESTAMP('2024-11-07 22:03:01.324', 'YYYY-MM-DD HH24:MI:SS.FF3') - | , 'pi' - | , 'alice' - | , 'bob' - | , 'siri' - | , 'alex' - | , 1.432 - | , 3.432 - | ) - |""".stripMargin) + statement.execute("CREATE TABLE TYPE_TEST(INT_COL INTEGER, NUM_8_COL NUMBER(8, 0), " + + " NUM_16_4_COL NUMBER(16, 4), DECIMAL_COL DECIMAL(8, 2), DATE_COL DATE, " + + "TIMESTAMP_COL TIMESTAMP, CHAR_COL CHAR(10), VARCHAR2_COL VARCHAR2(255), " + + "NCHAR_COL NCHAR(10), NCHAR2_COL NVARCHAR2(255), LONG_COL LONG, FLOAT_COL FLOAT, " + + "REAL_COL REAL)") + statement.execute("INSERT INTO TYPE_TEST( INT_COL, NUM_8_COL, NUM_16_4_COL," + + " DECIMAL_COL, DATE_COL, TIMESTAMP_COL, CHAR_COL, VARCHAR2_COL, NCHAR_COL, " + + "NCHAR2_COL, LONG_COL, FLOAT_COL, REAL_COL) " + + "VALUES ( 1, 2, 3.1415, 0.61, TO_DATE('2024-11-07', 'YYYY-MM-DD'), " + + "TO_TIMESTAMP('2024-11-07 22:03:01.324', 'YYYY-MM-DD HH24:MI:SS.FF3'), " + + "'pi', 'alice', 'bob', 'siri', 'alex', 1.432, 3.432)") val resultSet1 = statement.executeQuery("SELECT * FROM TYPE_TEST") while (resultSet1.next()) { From a7da4a646fe1a377d8f13e59a08020c76860764e Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:44:36 +0800 Subject: [PATCH 11/13] remove redundant impl of getTableTypesOperation in OracleSQLDialect.scala --- .../apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index 4394170c7c3..754b8d6b40a 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -80,10 +80,6 @@ class OracleSQLDialect extends JdbcDialect { query.toString() } - override def getTableTypesOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getColumnsQuery( session: Session, catalogName: String, From ccd7cae8bcadec8a4760e1409eec5ce291756cf8 Mon Sep 17 00:00:00 2001 From: naive-zhang Date: Tue, 26 Nov 2024 00:45:54 +0800 Subject: [PATCH 12/13] remove redundant override methods in OracleSQLDialect.scala --- .../engine/jdbc/dialect/OracleSQLDialect.scala | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index 754b8d6b40a..41f721a8bd8 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -108,18 +108,6 @@ class OracleSQLDialect extends JdbcDialect { query.toString() } - override def getFunctionsOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getPrimaryKeysOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getCrossReferenceOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getTRowSetGenerator(): JdbcTRowSetGenerator = new OracleTRowSetGenerator override def getSchemaHelper(): SchemaHelper = { From 171d06b9e5a4c15cae67b0f7eb226d4add524f48 Mon Sep 17 00:00:00 2001 From: native-zhang Date: Fri, 29 Nov 2024 01:03:44 +0800 Subject: [PATCH 13/13] use another implementation of transform decimal into int, in engine instead of KyuubiBaseResultSet --- .../jdbc/dialect/OracleSQLDialect.scala | 2 -- .../jdbc/oracle/OracleTRowSetGenerator.scala | 27 +++++++++++++++++-- .../jdbc/oracle/OracleStatementSuite.scala | 4 +-- .../kyuubi/jdbc/hive/KyuubiBaseResultSet.java | 4 --- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala index 41f721a8bd8..31ced54cdd8 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/OracleSQLDialect.scala @@ -24,10 +24,8 @@ import scala.collection.mutable.ArrayBuffer import org.apache.commons.lang3.StringUtils -import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.engine.jdbc.oracle.{OracleSchemaHelper, OracleTRowSetGenerator} import org.apache.kyuubi.engine.jdbc.schema.{JdbcTRowSetGenerator, SchemaHelper} -import org.apache.kyuubi.operation.Operation import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala index 33f6b16f8d8..41b216fc8f5 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleTRowSetGenerator.scala @@ -16,6 +16,29 @@ */ package org.apache.kyuubi.engine.jdbc.oracle -import org.apache.kyuubi.engine.jdbc.schema.DefaultJdbcTRowSetGenerator +import java.sql.Types -class OracleTRowSetGenerator extends DefaultJdbcTRowSetGenerator {} +import org.apache.kyuubi.engine.jdbc.schema.{Column, DefaultJdbcTRowSetGenerator} +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.{TColumn, TColumnValue} + +class OracleTRowSetGenerator extends DefaultJdbcTRowSetGenerator { + + override def toIntegerTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = { + asIntegerTColumn(rows, ordinal, (rows, ordinal) => Integer.parseInt(rows(ordinal).toString)) + } + + override def toIntegerTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = { + asIntegerTColumnValue(row, ordinal, x => Integer.parseInt(x.toString)) + super.toIntegerTColumnValue(row, ordinal) + } + + override def getColumnType(schema: Seq[Column], ordinal: Int): Int = { + schema(ordinal).sqlType match { + case Types.NUMERIC if schema(ordinal).scale == 0 => + Types.INTEGER + case Types.NUMERIC => + Types.DECIMAL + case _ => super.getColumnType(schema, ordinal) + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala index 91e9b0ad835..21d95173db6 100644 --- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/oracle/OracleStatementSuite.scala @@ -26,8 +26,8 @@ class OracleStatementSuite extends WithOracleEngine with HiveJDBCTestHelper { withJdbcStatement() { statement => statement.execute("create table T_TEST (ID INTEGER not null " + "constraint \"T_TEST_PK\" primary key, NAME VARCHAR2(64), USER_ID LONG, SCORE NUMBER(8,2))") - statement.execute("""INSERT INTO T_TEST(ID, NAME, USER_ID, SCORE) - |VALUES (1, 'Bob', 43254353,89.92)""".stripMargin) + statement.execute("INSERT INTO T_TEST(ID, NAME, USER_ID, SCORE) " + + "VALUES (1, 'Bob', 43254353,89.92)") val resultSet = statement.executeQuery("SELECT * FROM T_TEST") while (resultSet.next()) { val id = resultSet.getObject(1) diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java index 6d08236c115..cf47e104295 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiBaseResultSet.java @@ -338,10 +338,6 @@ private Object evaluate(TTypeId columnType, Object value) { if (value instanceof String) { return ((String) value).getBytes(); } - case INT_TYPE: - if (value instanceof String) { - return Integer.valueOf((String) value); - } return value; case TIMESTAMP_TYPE: return Timestamp.valueOf((String) value);