diff --git a/samples/arrow-flight-sql/java/README.md b/samples/arrow-flight-sql/java/README.md new file mode 100644 index 00000000000000..1eb574e2ab6626 --- /dev/null +++ b/samples/arrow-flight-sql/java/README.md @@ -0,0 +1,35 @@ + + +# How to use: + + 1. mvn clean install -U + 2. mvn package + 3. java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -cp java-0.1.jar doris.arrowflight.demo.Main "sql" "fe_ip" "fe_arrow_flight_port" "fe_query_port" + +# What can this demo do: + + This is a java demo for doris arrow flight sql, you can use this to test various connection + methods for sending queries to the doris arrow flight server, help you understand how to use arrow flight sql + and test performance. You should install maven prior to run this demo. + +# Performance test + + Section 6.2 of https://github.com/apache/doris/issues/25514 is the performance test + results of the doris arrow flight sql using java. \ No newline at end of file diff --git a/samples/arrow-flight-sql/java/pom.xml b/samples/arrow-flight-sql/java/pom.xml new file mode 100644 index 00000000000000..695c01aa5eb4f4 --- /dev/null +++ b/samples/arrow-flight-sql/java/pom.xml @@ -0,0 +1,168 @@ + + + 4.0.0 + + doris.arrowflight.demo + java + 0.1 + + + 17 + 17 + UTF-8 + + 18.1.0 + 0.15.0 + 2.17.1 + + + + + org.apache.arrow.adbc + adbc-driver-jdbc + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-flight-sql + ${adbc.version} + + + mysql + mysql-connector-java + 8.0.33 + + + org.apache.arrow + arrow-memory-core + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + flight-core + ${arrow.version} + + + org.apache.arrow + flight-sql + ${arrow.version} + + + + org.apache.arrow.adbc + adbc-core + ${adbc.version} + + + org.apache.arrow.adbc + adbc-driver-manager + ${adbc.version} + + + org.apache.arrow.adbc + adbc-sql + ${adbc.version} + + + org.apache.arrow + flight-sql-jdbc-core + ${arrow.version} + + + junit + junit + 4.13.1 + test + + + org.junit.jupiter + junit-jupiter + RELEASE + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + ${maven.compiler.source} + ${maven.compiler.target} + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java new file mode 100644 index 00000000000000..73b6edc5b16b33 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.Field; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + + +/** + * Iterate over each batch in ArrowReader. + * ArrowReader is the iterator returned by ADBC Client when executing a query. + */ +public class ArrowBatchReader { + + @FunctionalInterface + public interface LoadArrowBatchFunc { + void load(ArrowReader reader) throws IOException; + } + + /** + * Print one row in VectorSchemaRoot, if the output format is incorrect, may need to modify + * the output method of different types of ValueVector. + */ + public static void printRow(VectorSchemaRoot root, int rowIndex) { + if (root == null || rowIndex < 0 || rowIndex >= root.getRowCount()) { + System.out.println("Invalid row index: " + rowIndex); + return; + } + + System.out.print("> "); + for (Field field : root.getSchema().getFields()) { + ValueVector vector = root.getVector(field.getName()); + if (vector != null) { + if (vector instanceof org.apache.arrow.vector.DateDayVector) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + int dayOffset = ((org.apache.arrow.vector.DateDayVector) vector).get(rowIndex); + LocalDate date = LocalDate.ofEpochDay(dayOffset); + System.out.print(date.format(formatter)); + } else if (vector instanceof org.apache.arrow.vector.BitVector) { + System.out.print(((org.apache.arrow.vector.BitVector) vector).get(rowIndex) == 1); + } else { + // other types field + System.out.print(vector.getObject(rowIndex).toString()); + } + System.out.print(", "); + } + } + System.out.println(); + } + + /** + * Iterate over each batch in ArrowReader with the least cost, only record the number of rows and batches, + * usually used to test performance. + */ + public static LoadArrowBatchFunc loadArrowBatch = reader -> { + int rowCount = 0; + int batchCount = 0; + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + if (batchCount == 0) { + System.out.println("> " + root.getSchema().toString()); + printRow(root, 1); // only print first line + } + rowCount += root.getRowCount(); + batchCount += 1; + } + System.out.println("> batchCount: " + batchCount + ", rowCount: " + rowCount); + }; + + /** + * Iterate over each batch in ArrowReader and convert the batch to String, this will take more time. + */ + public static LoadArrowBatchFunc loadArrowBatchToString = reader -> { + int rowCount = 0; + List result = new ArrayList<>(); + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + if (result.size() == 0) { + System.out.println("> " + root.getSchema().toString()); + printRow(root, 0); // only print first line + } + result.add(root.contentToTSVString()); + rowCount += root.getRowCount(); + } + System.out.println("> batchCount: " + result.size() + ", rowCount: " + rowCount); + }; +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java new file mode 100644 index 00000000000000..fcfde19cf9470a --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +public class Configuration { + public String sql = ""; // require + public String ip = "127.0.0.1"; // require + public String arrowFlightPort = "9090"; // require + public String mysqlPort = "9030"; + public int retryTimes = 2; // The first execution is cold run + public String user = "root"; + public String password = ""; + + Configuration(String[] args) { + for (int i = 0; i < args.length; i++) { + switch (i) { + case 0 -> sql = args[i]; + case 1 -> ip = args[i]; + case 2 -> arrowFlightPort = args[i]; + case 3 -> mysqlPort = args[i]; + case 4 -> retryTimes = Integer.parseInt(args[i]); + case 5 -> user = args[i]; + case 6 -> password = args[i]; + } + } + } +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java new file mode 100644 index 00000000000000..b4a5f6bc7cb147 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import doris.arrowflight.demo.ArrowBatchReader.LoadArrowBatchFunc; +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcDatabase; +import org.apache.arrow.adbc.core.AdbcDriver; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.adbc.core.AdbcStatement.QueryResult; +import org.apache.arrow.adbc.driver.flightsql.FlightSqlDriver; +import org.apache.arrow.flight.Location; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; + +import java.util.HashMap; +import java.util.Map; + +/** + * Use the Arrow Flight ADBC driver to connect to the Doris Arrow Flight server and execute query. + */ +public class FlightAdbcDriver { + private static void connectAndExecute(Configuration configuration, LoadArrowBatchFunc loadArrowReader) { + final BufferAllocator allocator = new RootAllocator(); + FlightSqlDriver driver = new FlightSqlDriver(allocator); + Map parameters = new HashMap<>(); + AdbcDriver.PARAM_URI.set(parameters, + Location.forGrpcInsecure(configuration.ip, Integer.parseInt(configuration.arrowFlightPort)).getUri() + .toString()); + AdbcDriver.PARAM_USERNAME.set(parameters, configuration.user); + AdbcDriver.PARAM_PASSWORD.set(parameters, configuration.password); + + try { + AdbcDatabase adbcDatabase = driver.open(parameters); + AdbcConnection connection = adbcDatabase.connect(); + AdbcStatement stmt = connection.createStatement(); + long start = System.currentTimeMillis(); + stmt.setSqlQuery(configuration.sql); + + // executeQuery, two steps: + // 1. Execute Query and get returned FlightInfo; + // 2. Create FlightInfoReader to sequentially traverse each Endpoint; + QueryResult queryResult = stmt.executeQuery(); + ArrowReader reader = queryResult.getReader(); + loadArrowReader.load(reader); + System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start)); + + reader.close(); + queryResult.close(); + stmt.close(); + connection.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void run(Configuration configuration) { + System.out.println("*************************************"); + System.out.println("| FlightAdbcDriver |"); + System.out.println("*************************************"); + + System.out.println("FlightAdbcDriver > loadArrowBatch"); + connectAndExecute(configuration, ArrowBatchReader.loadArrowBatch); + System.out.println("FlightAdbcDriver > loadArrowBatchToString"); + connectAndExecute(configuration, ArrowBatchReader.loadArrowBatchToString); + } + + public static void main(String[] args) throws Exception { + Configuration configuration = new Configuration(args); + for (int i = 0; i < configuration.retryTimes; i++) { + run(configuration); + } + } +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java new file mode 100644 index 00000000000000..a376978628373d --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import doris.arrowflight.demo.ArrowBatchReader.LoadArrowBatchFunc; +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcDatabase; +import org.apache.arrow.adbc.core.AdbcDriver; +import org.apache.arrow.adbc.core.AdbcStatement; +import org.apache.arrow.adbc.driver.jdbc.JdbcDriver; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; + +import java.util.HashMap; +import java.util.Map; + +/** + * Use the Arrow Flight JDBC driver to connect to the Doris Arrow Flight server and execute query. + * Unlike the Java JDBC DriverManager, this is a JDBC Driver provided by Arrow Flight, which may contain + * some optimizations (although no performance advantage was observed). + */ +public class FlightJdbcDriver { + private static void connectAndExecute(Configuration configuration, LoadArrowBatchFunc loadArrowReader) { + String DB_URL = "jdbc:arrow-flight-sql://" + configuration.ip + ":" + configuration.arrowFlightPort + + "?useServerPrepStmts=false" + "&cachePrepStmts=true&useSSL=false&useEncryption=false"; + final Map parameters = new HashMap<>(); + AdbcDriver.PARAM_URI.set(parameters, DB_URL); + AdbcDriver.PARAM_USERNAME.set(parameters, configuration.user); + AdbcDriver.PARAM_PASSWORD.set(parameters, configuration.password); + + try { + BufferAllocator allocator = new RootAllocator(); + AdbcDatabase db = new JdbcDriver(allocator).open(parameters); + AdbcConnection connection = db.connect(); + AdbcStatement stmt = connection.createStatement(); + + long start = System.currentTimeMillis(); + stmt.setSqlQuery(configuration.sql); + AdbcStatement.QueryResult queryResult = stmt.executeQuery(); + ArrowReader reader = queryResult.getReader(); + loadArrowReader.load(reader); + System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start)); + + reader.close(); + queryResult.close(); + stmt.close(); + connection.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void run(Configuration configuration) { + System.out.println("*************************************"); + System.out.println("| FlightJdbcDriver |"); + System.out.println("*************************************"); + + System.out.println("FlightJdbcDriver > loadArrowBatch"); + connectAndExecute(configuration, ArrowBatchReader.loadArrowBatch); + System.out.println("FlightJdbcDriver > loadArrowBatchToString"); + connectAndExecute(configuration, ArrowBatchReader.loadArrowBatchToString); + } + + public static void main(String[] args) { + Configuration configuration = new Configuration(args); + for (int i = 0; i < configuration.retryTimes; i++) { + run(configuration); + } + } +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java new file mode 100644 index 00000000000000..995f59bc929d9b --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.flight.auth2.BearerCredentialWriter; +import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.sql.impl.FlightSql.TicketStatementQuery; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * Manually execute Arrow Flight SQL Rpc process, usually used for debug. + */ +public class FlightSqlClient { + public record FlightInfoResult(T first, T2 second, T3 third) { + } + + public record DummyFlightInfoResult(T first, T2 second) { + } + + /** + * Connect to FE Arrow Flight Server to obtain Bearertoken and execute Query to get Ticket. + */ + public static FlightInfoResult getFlightInfoFromDorisFe( + Configuration configuration) throws URISyntaxException { + BufferAllocator allocatorFE = new RootAllocator(Integer.MAX_VALUE); + final Location clientLocationFE = new Location( + new URI("grpc", null, configuration.ip, Integer.parseInt(configuration.arrowFlightPort), + null, null, null)); + FlightClient clientFE = FlightClient.builder(allocatorFE, clientLocationFE).build(); + org.apache.arrow.flight.sql.FlightSqlClient sqlClinetFE = new org.apache.arrow.flight.sql.FlightSqlClient( + clientFE); + + // Use username and password authentication to obtain a Bearertoken for subsequent access to the Doris Arrow Flight Server. + CredentialCallOption credentialCallOption = clientFE.authenticateBasicToken(configuration.user, + configuration.password).get(); + final org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement preparedStatement = sqlClinetFE.prepare( + configuration.sql, + credentialCallOption); + final FlightInfo info = preparedStatement.execute(credentialCallOption); + return new FlightInfoResult<>(info, credentialCallOption, preparedStatement); + } + + /** + * Use the correct Bearertoken and the correct Ticket, and the expected return result is normal. + */ + public static void getResultFromDorisBe(FlightInfo info, Ticket ticket, CredentialCallOption credentialCallOption) + throws Exception { + final Location locationBE = info.getEndpoints().get(0).getLocations().get(0); + // 连接 BE Arrow Flight Server + BufferAllocator allocatorBE = new RootAllocator(Integer.MAX_VALUE); + FlightClient clientBE = FlightClient.builder(allocatorBE, locationBE).build(); + org.apache.arrow.flight.sql.FlightSqlClient sqlClinetBE = new org.apache.arrow.flight.sql.FlightSqlClient( + clientBE); + + FlightStream stream = sqlClinetBE.getStream(ticket, credentialCallOption); + int rowCount = 0; + int batchCount = 0; + while (stream.next()) { + VectorSchemaRoot root = stream.getRoot(); + if (batchCount == 0) { + System.out.println("> " + root.getSchema().toString()); + ArrowBatchReader.printRow(root, 1); // only print first line + } + rowCount += root.getRowCount(); + batchCount += 1; + } + System.out.println("> batchCount: " + batchCount + ", rowCount: " + rowCount); + stream.close(); + } + + /** + * Construct a dummy Ticket and CredentialCallOption to simulate the BE Arrow Flight Server being hacked, + * to analyze data security. + * + * @return get error `INVALID_ARGUMENT: Malformed ticket` + */ + public static DummyFlightInfoResult constructDummyFlightInfo() { + String Bearertoken = "ojatddjr72k1ss20sqkatkhtd7"; + String queryId = "18c64b4e15094922-af5fea3da80fb89f"; + String query = "select * from clickbench.hits limit 10;"; + CredentialCallOption dummyCredentialCallOption = new CredentialCallOption( + new BearerCredentialWriter(Bearertoken)); + final ByteString handle = ByteString.copyFromUtf8(queryId + ":" + query); + TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder().setStatementHandle(handle).build(); + final Ticket dummyTicket = new Ticket(Any.pack(ticketStatement).toByteArray()); + return new DummyFlightInfoResult<>(dummyTicket, dummyCredentialCallOption); + } + + public static void run(Configuration configuration) { + System.out.println("*************************************"); + System.out.println("| FlightSqlClient |"); + System.out.println("*************************************"); + + try { + System.out.println("FlightSqlClient > getFlightInfoFromDorisFe"); + var flightInfo = getFlightInfoFromDorisFe(configuration); + getResultFromDorisBe(flightInfo.first(), flightInfo.first().getEndpoints().get(0).getTicket(), + flightInfo.second()); + System.out.println(); + + System.out.println( + "FlightSqlClient > constructDummyFlightInfo, don't be afraid! expected to get error `INVALID_ARGUMENT: Malformed ticket`"); + var dummyFlightInfo = constructDummyFlightInfo(); + getResultFromDorisBe(flightInfo.first(), dummyFlightInfo.first(), dummyFlightInfo.second()); + System.out.println(); + + flightInfo.third().close(flightInfo.second()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + Configuration configuration = new Configuration(args); + run(configuration); + } +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java new file mode 100644 index 00000000000000..78b888538ea1d4 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import doris.arrowflight.demo.JdbcResultSetReader.LoadJdbcResultSetFunc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Objects; + +/** + * Use the Java JDBC DriverManager to connect to the Doris Arrow Flight server and execute query. + * Usually, DriverManager is used to connect to the database using the Mysql protocol in Java. only need to replace + * `jdbc:mysql` in the URI with `jdbc:arrow-flight-sql` to connect to the database using the Arrow Flight SQL protocol + * (provided that the database implements the Arrow Flight server). + */ +public class JdbcDriverManager { + private static void connectAndExecute(Configuration configuration, String urlPrefix, String port, + LoadJdbcResultSetFunc loadJdbcResultSetFunc) { + String DB_URL = urlPrefix + "://" + configuration.ip + ":" + port + "?useServerPrepStmts=false" + + "&cachePrepStmts=true&useSSL=false&useEncryption=false"; + try { + long start = System.currentTimeMillis(); + Connection conn = DriverManager.getConnection(DB_URL, configuration.user, configuration.password); + Statement stmt = conn.createStatement(); + stmt.execute(configuration.sql); + + final ResultSet resultSet = stmt.getResultSet(); + loadJdbcResultSetFunc.load(resultSet); + System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start)); + + stmt.close(); + conn.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void run(Configuration configuration) { + System.out.println("*************************************"); + System.out.println("| JdbcDriverManager |"); + System.out.println("*************************************"); + + try { + if (!Objects.equals(configuration.mysqlPort, "")) { + Class.forName("com.mysql.cj.jdbc.Driver"); + System.out.println("JdbcDriverManager > jdbc:mysql > loadJdbcResult"); + connectAndExecute(configuration, "jdbc:mysql", configuration.mysqlPort, + JdbcResultSetReader.loadJdbcResult); + System.out.println("JdbcDriverManager > jdbc:mysql > loadJdbcResultToString"); + connectAndExecute(configuration, "jdbc:mysql", configuration.mysqlPort, + JdbcResultSetReader.loadJdbcResultToString); + } + + Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver"); + System.out.println("JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString"); + connectAndExecute(configuration, "jdbc:arrow-flight-sql", configuration.arrowFlightPort, + JdbcResultSetReader.loadJdbcResult); + System.out.println("JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString"); + connectAndExecute(configuration, "jdbc:arrow-flight-sql", configuration.arrowFlightPort, + JdbcResultSetReader.loadJdbcResultToString); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) throws ClassNotFoundException { + Configuration configuration = new Configuration(args); + for (int i = 0; i < configuration.retryTimes; i++) { + run(configuration); + } + } +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java new file mode 100644 index 00000000000000..221fbb239cc703 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * Iterate over each row in jdbc ResultSet. + */ +public class JdbcResultSetReader { + + @FunctionalInterface + public interface LoadJdbcResultSetFunc { + void load(ResultSet resultSet) throws IOException, SQLException; + } + + public static LoadJdbcResultSetFunc loadJdbcResult = resultSet -> { + int rowCount = 0; + final int columnCount = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + rowCount += 1; + } + System.out.println("> rowCount: " + rowCount + ", columnCount: " + columnCount); + }; + + public static LoadJdbcResultSetFunc loadJdbcResultToString = resultSet -> { + int rowCount = 0; + final int columnCount = resultSet.getMetaData().getColumnCount(); + List result = new ArrayList<>(); + while (resultSet.next()) { + StringBuilder line = new StringBuilder(); + for (int i = 1; i <= columnCount; i++) { + line.append(resultSet.getString(i)).append(","); + } + if (rowCount == 0) { // only print first line + System.out.println("> " + line); + } + rowCount += 1; + result.add(line.toString()); + } + System.out.println( + "> rowCount: " + rowCount + ", columnCount: " + columnCount + " resultSize: " + result.size()); + }; +} diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java new file mode 100644 index 00000000000000..763a380fbc3911 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +public class Main { + public static void main(String[] args) { + Configuration configuration = new Configuration(args); + for (int i = 0; i < configuration.retryTimes; i++) { + FlightAdbcDriver.run(configuration); + FlightJdbcDriver.run(configuration); + JdbcDriverManager.run(configuration); + } + FlightSqlClient.run(configuration); + } +} diff --git a/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java b/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java new file mode 100644 index 00000000000000..a6582fae4f2ac8 --- /dev/null +++ b/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +package doris.arrowflight.demo; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; + +/** + * Unit test for simple App. + */ +public class ConfigurationTest { + + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() { + assertTrue(true); + } +}