From 87e948a80c363289f9bbf43d9150f8ca11ef258e Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 4 Nov 2024 17:54:25 +0800 Subject: [PATCH 01/27] add impl Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 21 ++ flex/CMakeLists.txt | 10 + flex/planner/CMakeLists.txt | 5 + flex/planner/README.md | 3 + flex/planner/graph_planner.cc | 235 ++++++++++++++++++ flex/planner/graph_planner.h | 110 ++++++++ flex/tests/CMakeLists.txt | 1 + flex/tests/planner/CMakeLists.txt | 2 + flex/tests/planner/graph_planner_test.cc | 50 ++++ .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 2 +- .../ir/meta/fetcher/StaticIrMetaFetcher.java | 2 +- .../common/ir/tools/GraphPlanner.java | 27 +- 12 files changed, 458 insertions(+), 10 deletions(-) create mode 100644 flex/planner/CMakeLists.txt create mode 100644 flex/planner/README.md create mode 100644 flex/planner/graph_planner.cc create mode 100644 flex/planner/graph_planner.h create mode 100644 flex/tests/planner/CMakeLists.txt create mode 100644 flex/tests/planner/graph_planner_test.cc diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 279603cbca35..4cfe963c35f0 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -317,6 +317,27 @@ jobs: cd ${GITHUB_WORKSPACE}/interactive_engine/ mvn clean install -Pexperimental -DskipTests -q + - name: Test physical plan generation + run: + cd ${GITHUB_WORKSPACE}/flex/build + cat > /tmp/physical_plan_gen_config.yaml <> /tmp/physical_plan_gen_config.yaml + echo " meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml + + ./tests/planner/graph_planner_test ${GITHUB_WORKSPACE}/interactive_engine/compiler/target/compiler-0.0.1-SNAPSHOT.jar:${GITHUB_WORKSPACE}/interactive_engine/compiler/target/libs/ \ + ${GITHUB_WORKSPACE}/interactive_engine/executor/ir/target/release/ ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml \ + /tmp/physical_plan_gen_config.yaml "MATCH(n) return count(n);" + - name: Run End-to-End cypher adhoc ldbc query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index c0385658cec2..4439aece78e5 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -93,6 +93,15 @@ find_package(libgrapelite REQUIRED) include_directories(SYSTEM ${LIBGRAPELITE_INCLUDE_DIRS}) include_directories(SYSTEM ${LIBGRAPELITE_INCLUDE_DIRS}/grape/analytical_apps) +# find JNI----------------------------------------------------------------------- +find_package(JNI REQUIRED) +if (JNI_FOUND) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH2}) +else() + message(FATAL_ERROR "JNI not found") +endif() + # find Threads------------------------------------------------------------------ set(CMAKE_THREAD_PREFER_PTHREAD ON) find_package(Threads REQUIRED) @@ -233,6 +242,7 @@ endif() if (BUILD_TEST) add_subdirectory(tests) endif() +add_subdirectory(planner) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flex-config.in.cmake diff --git a/flex/planner/CMakeLists.txt b/flex/planner/CMakeLists.txt new file mode 100644 index 000000000000..ada8073abe65 --- /dev/null +++ b/flex/planner/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE PLANNER_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") + +add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) +target_link_libraries(graph_planner PUBLIC flex_plan_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES}) +target_include_directories(graph_planner PUBLIC $) \ No newline at end of file diff --git a/flex/planner/README.md b/flex/planner/README.md new file mode 100644 index 000000000000..cae277f0e1f3 --- /dev/null +++ b/flex/planner/README.md @@ -0,0 +1,3 @@ +# GraphScope Interactive Query Planner + +This directory contains the c++ wrapper of Interactive Query Planner, which is implemented in [java](https://github.com/alibaba/GraphScope/tree/main/interactive_engine/compiler). \ No newline at end of file diff --git a/flex/planner/graph_planner.cc b/flex/planner/graph_planner.cc new file mode 100644 index 000000000000..deca274d3f69 --- /dev/null +++ b/flex/planner/graph_planner.cc @@ -0,0 +1,235 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include + +namespace gs { +namespace jni { + +static JavaVM* _jvm = NULL; + +JavaVM* CreateJavaVM(const std::string& jvm_options) { + const char *p, *q; + const char* jvm_opts; + if (jvm_options.empty()) { + jvm_opts = getenv("FLEX_JVM_OPTS"); + } else { + jvm_opts = jvm_options.c_str(); + } + if (jvm_opts == NULL) { + LOG(FATAL) << "Expect FLEX_JVM_OPTS set before initiate jvm"; + return NULL; + } + VLOG(1) << "Jvm opts str: " << jvm_opts; + + if (*jvm_opts == '\0') + return NULL; + + int num_of_opts = 1; + for (const char* p = jvm_opts; *p; p++) { + if (*p == ' ') + num_of_opts++; + } + + if (num_of_opts == 0) + return NULL; + + JavaVM* jvm = NULL; + JNIEnv* env = NULL; + int i = 0; + int status = 1; + JavaVMInitArgs vm_args; + + JavaVMOption* options = new JavaVMOption[num_of_opts]; + memset(options, 0, sizeof(JavaVMOption) * num_of_opts); + + for (p = q = jvm_opts;; p++) { + if (*p == ' ' || *p == '\0') { + if (q >= p) { + goto ret; + } + char* opt = new char[p - q + 1]; + memcpy(opt, q, p - q); + opt[p - q] = '\0'; + options[i++].optionString = opt; + q = p + 1; // assume opts are separated by single space + if (*p == '\0') + break; + } + } + + memset(&vm_args, 0, sizeof(vm_args)); + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = num_of_opts; + vm_args.options = options; + + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + if (status == JNI_OK) { + LOG(INFO) << "Create java virtual machine successfully."; + } else if (status == JNI_EEXIST) { + VLOG(1) << "JNI evn already exists."; + } else { + LOG(ERROR) << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")\n"; + } + +ret: + for (int i = 0; i < num_of_opts; i++) { + delete[] options[i].optionString; + } + delete[] options; + return jvm; +} + +// One process can only create jvm for once. +JavaVM* GetJavaVM(const std::string jvm_options = "") { + if (_jvm == NULL) { + // Try to find whether there exists one javaVM + jsize nVMs; + JNI_GetCreatedJavaVMs(NULL, 0, + &nVMs); // 1. just get the required array length + VLOG(1) << "Found " << nVMs << " VMs existing in this process."; + JavaVM** buffer = new JavaVM*[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + for (auto i = 0; i < nVMs; ++i) { + if (buffer[i] != NULL) { + _jvm = buffer[i]; + VLOG(1) << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm); + return _jvm; + } + } + _jvm = CreateJavaVM(jvm_options); + VLOG(1) << "Created JVM " << reinterpret_cast(_jvm); + } + return _jvm; +} + +JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + +JNIEnvMark::JNIEnvMark(const std::string& jvm_options) : _env(NULL) { + if (!GetJavaVM(jvm_options)) { + return; + } + int status = + GetJavaVM(jvm_options) + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + if (status != JNI_OK) { + LOG(ERROR) << "Error attach current thread: " << status; + } +} + +JNIEnvMark::~JNIEnvMark() { + if (_env) { + GetJavaVM()->DetachCurrentThread(); + } +} + +JNIEnv* JNIEnvMark::env() { return _env; } + +physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( + const std::string& compiler_config_path, + const std::string& cypher_query_string) { + physical::PhysicalPlan physical_plan; + if (!is_valid()) { + LOG(ERROR) << "Invalid GraphPlannerWrapper."; + return physical_plan; + } + jstring param1 = + jni_wrapper_.env()->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = + jni_wrapper_.env()->NewStringUTF(cypher_query_string.c_str()); + + jbyteArray res = (jbyteArray) jni_wrapper_.env()->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2); + if (jni_wrapper_.env()->ExceptionCheck()) { + jni_wrapper_.env()->ExceptionDescribe(); + jni_wrapper_.env()->ExceptionClear(); + LOG(ERROR) << "Error in calling GraphPlanner."; + return physical_plan; + } + if (res == NULL) { + LOG(ERROR) << "Fail to generate plan."; + return physical_plan; + } + jbyte* str = jni_wrapper_.env()->GetByteArrayElements(res, NULL); + jsize len = jni_wrapper_.env()->GetArrayLength(res); + LOG(INFO) << "Physical plan size: " << len; + + physical_plan.ParseFromArray(str, len); + jni_wrapper_.env()->ReleaseByteArrayElements(res, str, 0); + jni_wrapper_.env()->DeleteLocalRef(param1); + jni_wrapper_.env()->DeleteLocalRef(param2); + jni_wrapper_.env()->DeleteLocalRef(res); + + return physical_plan; +} + +std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json) { + auto expanded_java_path = expand_directory(java_path); + VLOG(10) << "Expanded java path: " << expanded_java_path; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + if (!graph_statistic_json.empty()) { + jvm_options += " -Dgraph.statistic=" + graph_statistic_json; + } + return jvm_options; +} + +std::string GraphPlannerWrapper::expand_directory(const std::string& path) { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) { + auto sub_path = path.substr(start, end - start); + if (!sub_path.empty()) { + if (std::filesystem::is_directory(sub_path)) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } else { + paths.push_back(sub_path); + } + } + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + if (!sub_path.empty()) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } + std::stringstream ss; + for (const auto& p : paths) { + ss << p << ":"; + } + return ss.str(); +} + +std::vector GraphPlannerWrapper::list_files( + const std::string& path) { + // list all files in the directory + std::vector files; + for (const auto& entry : std::filesystem::directory_iterator(path)) { + files.push_back(entry.path().string()); + } + return files; +} + +} // namespace jni +} // namespace gs diff --git a/flex/planner/graph_planner.h b/flex/planner/graph_planner.h new file mode 100644 index 000000000000..1372990758ff --- /dev/null +++ b/flex/planner/graph_planner.h @@ -0,0 +1,110 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#ifndef PLANNER_GRAPH_PLANNER_H_ +#define PLANNER_GRAPH_PLANNER_H_ + +#include +#include +#include +#include +#include + +#include "flex/proto_generated_gie/physical.pb.h" + +#include "glog/logging.h" + +namespace gs { +namespace jni { +struct JNIEnvMark { + JNIEnv* _env; + + JNIEnvMark(); + JNIEnvMark(const std::string& jvm_options); + ~JNIEnvMark(); + JNIEnv* env(); +}; + +class GraphPlannerWrapper { + public: + static constexpr const char* kGraphPlannerClass = + "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; + static constexpr const char* kGraphPlannerMethod = "generatePhysicalPlan"; + static constexpr const char* kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;)[B"; + + GraphPlannerWrapper(const std::string java_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json = "") + : jni_wrapper_(generate_jvm_options( + java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { + jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); + if (clz == NULL) { + LOG(ERROR) << "Fail to find class: " << kGraphPlannerClass; + return; + } + graph_planner_clz_ = (jclass) jni_wrapper_.env()->NewGlobalRef(clz); + jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( + graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); + if (j_method_id == NULL) { + LOG(ERROR) << "Fail to find method: " << kGraphPlannerMethod; + return; + } + graph_planner_method_id_ = j_method_id; + } + + ~GraphPlannerWrapper() { + if (graph_planner_clz_ != NULL) { + jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); + } + } + + inline bool is_valid() { + return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; + } + + /** + * @brief Invoker GraphPlanner to generate a physical plan from a cypher + * query. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return physical plan in string. + */ + physical::PhysicalPlan CompilePlan(const std::string& compiler_config_path, + const std::string& cypher_query_string); + + private: + std::string generate_jvm_options(const std::string java_path, + const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json); + + // We need to list all files in the directory, if exists. + // The reason why we need to list all files in the directory is that + // java -Djava.class.path=dir/* (in jni, which we are using)will not load all + // jar files in the directory, While java -cp dir/* will load all jar files in + // the directory. + std::string expand_directory(const std::string& path); + std::vector list_files(const std::string& path); + + JNIEnvMark jni_wrapper_; + jclass graph_planner_clz_; + jmethodID graph_planner_method_id_; +}; + +} // namespace jni +} // namespace gs + +#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file diff --git a/flex/tests/CMakeLists.txt b/flex/tests/CMakeLists.txt index e2bb952d7eb9..d09a89e0ce9f 100644 --- a/flex/tests/CMakeLists.txt +++ b/flex/tests/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(hqps) add_subdirectory(rt_mutable_graph) +add_subdirectory(planner) \ No newline at end of file diff --git a/flex/tests/planner/CMakeLists.txt b/flex/tests/planner/CMakeLists.txt new file mode 100644 index 000000000000..0ca1c10a4884 --- /dev/null +++ b/flex/tests/planner/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(graph_planner_test graph_planner_test.cc) +target_link_libraries(graph_planner_test graph_planner ${GLOG_LIBRARIES}) \ No newline at end of file diff --git a/flex/tests/planner/graph_planner_test.cc b/flex/tests/planner/graph_planner_test.cc new file mode 100644 index 000000000000..fa4804ad9c88 --- /dev/null +++ b/flex/tests/planner/graph_planner_test.cc @@ -0,0 +1,50 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/planner/graph_planner.h" +#include +#include +#include + +int main(int argc, char** argv) { + if (argc != 6) { + LOG(ERROR) << "Usage: " << argv[0] + << " " + " "; + LOG(ERROR) << "But got " << argc << " arguments."; + return 1; + } + std::string java_path = argv[1]; + std::string jna_path = argv[2]; + std::string graph_schema_path = argv[3]; + std::string compiler_config_path = argv[4]; + std::string cypher_query = argv[5]; + if (java_path.empty() || jna_path.empty() || graph_schema_path.empty()) { + LOG(ERROR) << "Invalid input."; + return 1; + } + if (!std::filesystem::exists(compiler_config_path)) { + LOG(ERROR) << "Invalid compiler config path."; + return 1; + } + gs::jni::GraphPlannerWrapper planner(java_path, jna_path, graph_schema_path); + if (!planner.is_valid()) { + LOG(ERROR) << "Invalid GraphPlannerWrapper."; + return 1; + } + auto plan = planner.CompilePlan(compiler_config_path, cypher_query); + CHECK(plan.plan_size() == 3) << "Invalid plan size: " << plan.plan_size(); + return 0; +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index 931e78aae887..1d41de91b158 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -137,7 +137,7 @@ private synchronized void syncStats(boolean statsEnabled) { } } } catch (Throwable e) { - logger.warn("failed to read graph statistics, error is {}", e); + logger.warn("failed to read graph statistics, error is: " + e); } finally { try { if (this.currentState != null diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java index b48cc07d81a1..2afd866ad484 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java @@ -51,7 +51,7 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throw try { return this.reader.readStats(meta.getGraphId()); } catch (Exception e) { - logger.warn("failed to read graph statistics, error is {}", e); + logger.warn("failed to read graph statistics, error is: " + e); return null; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 24c821b2f2e9..b38be3adf8f0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -221,6 +221,24 @@ private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker "unknown graph meta reader mode: " + schemaUri.getScheme()); } + public static Summary generatePlan(String config_path, String query_string) throws Exception { + Configs configs = Configs.Factory.create(config_path); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + PlannerInstance instance = planner.instance(query_string, metaFetcher.fetch().get()); + Summary summary = instance.plan(); + return summary; + } + + public static byte[] generatePhysicalPlan(String config_path, String query_string) throws Exception { + Summary summary = generatePlan(config_path, query_string); + PhysicalPlan physicalPlan = summary.physicalPlan; + return physicalPlan.getContent(); + } + public static void main(String[] args) throws Exception { if (args.length < 4 || args[0].isEmpty() @@ -232,15 +250,8 @@ public static void main(String[] args) throws Exception { + " '' ''" + " 'optional '"); } - Configs configs = Configs.Factory.create(args[0]); - GraphRelOptimizer optimizer = - new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); - IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); - GraphPlanner planner = - new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); - PlannerInstance instance = planner.instance(query, metaFetcher.fetch().get()); - Summary summary = instance.plan(); + Summary summary = generatePlan(args[0], query); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; FileUtils.writeByteArrayToFile(new File(args[2]), physicalPlan.getContent()); From ba68cb51cdaf0dca75c124fbd9320762b6b8b6f3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 4 Nov 2024 19:17:59 +0800 Subject: [PATCH 02/27] call-compiler-via-jni Committed-by: xiaolei.zl from Dev container --- .../com/alibaba/graphscope/common/ir/tools/GraphPlanner.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index b38be3adf8f0..239592bd3b9b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -233,7 +233,8 @@ public static Summary generatePlan(String config_path, String query_string) thro return summary; } - public static byte[] generatePhysicalPlan(String config_path, String query_string) throws Exception { + public static byte[] generatePhysicalPlan(String config_path, String query_string) + throws Exception { Summary summary = generatePlan(config_path, query_string); PhysicalPlan physicalPlan = summary.physicalPlan; return physicalPlan.getContent(); From ef312c9450b949b121f7ca197fbddaaf9aa3f222 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 4 Nov 2024 19:54:59 +0800 Subject: [PATCH 03/27] fixing ci Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 4cfe963c35f0..39c86bae49e8 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -318,7 +318,7 @@ jobs: mvn clean install -Pexperimental -DskipTests -q - name: Test physical plan generation - run: + run: | cd ${GITHUB_WORKSPACE}/flex/build cat > /tmp/physical_plan_gen_config.yaml < Date: Fri, 8 Nov 2024 11:04:26 +0800 Subject: [PATCH 04/27] add guard to control jni or subprocess Committed-by: xiaolei.zl from Dev container --- flex/planner/graph_planner.cc | 207 +++++++++++++++++------ flex/planner/graph_planner.h | 45 ++++- flex/tests/planner/graph_planner_test.cc | 2 +- 3 files changed, 197 insertions(+), 57 deletions(-) diff --git a/flex/planner/graph_planner.cc b/flex/planner/graph_planner.cc index deca274d3f69..3605722cd21c 100644 --- a/flex/planner/graph_planner.cc +++ b/flex/planner/graph_planner.cc @@ -12,10 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include +#include +#include +#include #include namespace gs { +#if (GRAPH_PLANNER_JNI_INVOKER) namespace jni { static JavaVM* _jvm = NULL; @@ -139,11 +144,72 @@ JNIEnvMark::~JNIEnvMark() { JNIEnv* JNIEnvMark::env() { return _env; } -physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( - const std::string& compiler_config_path, - const std::string& cypher_query_string) { +} // namespace jni + +#endif + +std::vector list_files(const std::string& path) { + // list all files in the directory + std::vector files; + for (const auto& entry : std::filesystem::directory_iterator(path)) { + files.push_back(entry.path().string()); + } + return files; +} + +std::string GraphPlannerWrapper::expand_directory(const std::string& path) { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) { + auto sub_path = path.substr(start, end - start); + if (!sub_path.empty()) { + if (std::filesystem::is_directory(sub_path)) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } else { + paths.push_back(sub_path); + } + } + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + if (!sub_path.empty()) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } + std::stringstream ss; + for (const auto& p : paths) { + ss << p << ":"; + } + return ss.str(); +} + +#if (GRAPH_PLANNER_JNI_INVOKER) + +std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json) { + auto expanded_java_path = expand_directory(java_path); + VLOG(10) << "Expanded java path: " << expanded_java_path; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + if (!graph_statistic_json.empty()) { + jvm_options += " -Dgraph.statistic=" + graph_statistic_json; + } + return jvm_options; +} + +physical::PhysicalPlan compilePlanJNI(jclass graph_planner_clz_, + jmethodID graph_planner_method_id_, + jni::JNIEnvMark& jni_wrapper_, + const std::string& compiler_config_path, + const std::string& cypher_query_string) { physical::PhysicalPlan physical_plan; - if (!is_valid()) { + if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { LOG(ERROR) << "Invalid GraphPlannerWrapper."; return physical_plan; } @@ -176,60 +242,101 @@ physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( return physical_plan; } +#endif -std::string GraphPlannerWrapper::generate_jvm_options( - const std::string java_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json) { - auto expanded_java_path = expand_directory(java_path); - VLOG(10) << "Expanded java path: " << expanded_java_path; - std::string jvm_options = "-Djava.class.path=" + expanded_java_path; - jvm_options += " -Djna.library.path=" + jna_path; - jvm_options += " -Dgraph.schema=" + graph_schema_yaml; - if (!graph_statistic_json.empty()) { - jvm_options += " -Dgraph.statistic=" + graph_statistic_json; +#if (!GRAPH_PLANNER_JNI_INVOKER) + +physical::PhysicalPlan readPhysicalPlan(const std::string& output_file) { + std::ifstream file(output_file, std::ios::binary); + + if (!file.is_open()) { + LOG(ERROR) << "Fail to open file: " << output_file; + return physical::PhysicalPlan(); } - return jvm_options; + + file.seekg(0, std::ios::end); + size_t size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::string buffer; + buffer.resize(size); + + file.read(&buffer[0], size); + + file.close(); + physical::PhysicalPlan plan; + if (!plan.ParseFromString(std::string(buffer))) { + LOG(ERROR) << "Fail to parse physical plan."; + return physical::PhysicalPlan(); + } + return plan; } -std::string GraphPlannerWrapper::expand_directory(const std::string& path) { - std::vector paths; - std::string::size_type start = 0; - std::string::size_type end = path.find(':'); - while (end != std::string::npos) { - auto sub_path = path.substr(start, end - start); - if (!sub_path.empty()) { - if (std::filesystem::is_directory(sub_path)) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } else { - paths.push_back(sub_path); - } +physical::PhysicalPlan compilePlanSubprocess( + const std::string& class_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json, + const std::string& compiler_config_path, + const std::string& cypher_query_string) { + // use execvp to run the command + auto random_prefix = std::to_string( + std::chrono::system_clock::now().time_since_epoch().count()); + std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; + std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; + std::ofstream query_file(dst_query_path); + query_file << cypher_query_string; + query_file.close(); + pid_t pid = fork(); + if (pid == 0) { + // generate a random file name for query + const char* const command_string_array[] = { + "java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + "com.alibaba.graphscope.common.ir.tools.GraphPlanner", + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; + LOG(INFO) << command_string_array[0] << " " << command_string_array[1] + << " " << command_string_array[2] << " " + << command_string_array[3] << " " << command_string_array[4] + << " " << command_string_array[5] << " " + << command_string_array[6] << " " << command_string_array[7] + << " " << command_string_array[8] << " " + << command_string_array[9]; + execvp(command_string_array[0], + const_cast(command_string_array)); + } else if (pid < 0) { + LOG(ERROR) << "Error in fork."; + } else { + int status; + waitpid(pid, &status, 0); + if (status != 0) { + LOG(ERROR) << "Error in running command."; } - start = end + 1; - end = path.find(':', start); + return readPhysicalPlan(dst_output_file); } - auto sub_path = path.substr(start); - if (!sub_path.empty()) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } - std::stringstream ss; - for (const auto& p : paths) { - ss << p << ":"; - } - return ss.str(); + return physical::PhysicalPlan(); } +#endif -std::vector GraphPlannerWrapper::list_files( - const std::string& path) { - // list all files in the directory - std::vector files; - for (const auto& entry : std::filesystem::directory_iterator(path)) { - files.push_back(entry.path().string()); - } - return files; +physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( + const std::string& compiler_config_path, + const std::string& cypher_query_string) { +#if (GRAPH_PLANNER_JNI_INVOKER) + return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, + jni_wrapper_, compiler_config_path, + cypher_query_string); +#else + return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, + graph_statistic_json_, compiler_config_path, + cypher_query_string); +#endif } -} // namespace jni } // namespace gs diff --git a/flex/planner/graph_planner.h b/flex/planner/graph_planner.h index 1372990758ff..68e2325b61a2 100644 --- a/flex/planner/graph_planner.h +++ b/flex/planner/graph_planner.h @@ -26,7 +26,15 @@ limitations under the License. #include "glog/logging.h" +// #define GRAPH_PLANNER_JNI_INVOKER JNI +#ifndef GRAPH_PLANNER_JNI_INVOKER +#define GRAPH_PLANNER_JNI_INVOKER 0 // 1: JNI, 0: subprocess +#endif +// Could be JNI or subprocess + namespace gs { + +#if (GRAPH_PLANNER_JNI_INVOKER) namespace jni { struct JNIEnvMark { JNIEnv* _env; @@ -37,6 +45,9 @@ struct JNIEnvMark { JNIEnv* env(); }; +} // namespace jni +#endif + class GraphPlannerWrapper { public: static constexpr const char* kGraphPlannerClass = @@ -48,6 +59,7 @@ class GraphPlannerWrapper { GraphPlannerWrapper(const std::string java_path, const std::string& jna_path, const std::string& graph_schema_yaml, const std::string& graph_statistic_json = "") +#if (GRAPH_PLANNER_JNI_INVOKER) : jni_wrapper_(generate_jvm_options( java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); @@ -64,15 +76,29 @@ class GraphPlannerWrapper { } graph_planner_method_id_ = j_method_id; } +#else + : jna_path_("-Djna.library.path=" + jna_path), + graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), + graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { + class_path_ = expand_directory(java_path); + } +#endif ~GraphPlannerWrapper() { +#if (GRAPH_PLANNER_JNI_INVOKER) if (graph_planner_clz_ != NULL) { jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); } +#endif } inline bool is_valid() { +#if (GRAPH_PLANNER_JNI_INVOKER) return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; +#else + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. +#endif } /** @@ -90,21 +116,28 @@ class GraphPlannerWrapper { const std::string& jna_path, const std::string& graph_schema_yaml, const std::string& graph_statistic_json); - + // physical::PhysicalPlan compilePlanJNI(const std::string& + // compiler_config_path, + // const std::string& + // cypher_query_string); + std::string expand_directory(const std::string& path); +#if (GRAPH_PLANNER_JNI_INVOKER) // We need to list all files in the directory, if exists. // The reason why we need to list all files in the directory is that // java -Djava.class.path=dir/* (in jni, which we are using)will not load all // jar files in the directory, While java -cp dir/* will load all jar files in // the directory. - std::string expand_directory(const std::string& path); - std::vector list_files(const std::string& path); - JNIEnvMark jni_wrapper_; + gs::jni::JNIEnvMark jni_wrapper_; jclass graph_planner_clz_; jmethodID graph_planner_method_id_; +#else + std::string class_path_; + std::string jna_path_; + std::string graph_schema_yaml_; + std::string graph_statistic_json_; +#endif }; - -} // namespace jni } // namespace gs #endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file diff --git a/flex/tests/planner/graph_planner_test.cc b/flex/tests/planner/graph_planner_test.cc index fa4804ad9c88..46231d9150f0 100644 --- a/flex/tests/planner/graph_planner_test.cc +++ b/flex/tests/planner/graph_planner_test.cc @@ -39,7 +39,7 @@ int main(int argc, char** argv) { LOG(ERROR) << "Invalid compiler config path."; return 1; } - gs::jni::GraphPlannerWrapper planner(java_path, jna_path, graph_schema_path); + gs::GraphPlannerWrapper planner(java_path, jna_path, graph_schema_path); if (!planner.is_valid()) { LOG(ERROR) << "Invalid GraphPlannerWrapper."; return 1; From 65b049e3e92b3eb2402f81b6e0a4d62f90c793b0 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 11 Nov 2024 10:54:54 +0800 Subject: [PATCH 05/27] support jni or pip3 Committed-by: xiaolei.zl from Dev container --- flex/planner/graph_planner.cc | 120 +++++++++++------- flex/planner/graph_planner.h | 4 +- flex/tests/planner/graph_planner_test.cc | 18 ++- .../common/ir/tools/GraphPlanner.java | 24 +++- 4 files changed, 111 insertions(+), 55 deletions(-) diff --git a/flex/planner/graph_planner.cc b/flex/planner/graph_planner.cc index 3605722cd21c..06913a1b76aa 100644 --- a/flex/planner/graph_planner.cc +++ b/flex/planner/graph_planner.cc @@ -12,11 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include +#include #include #include #include #include +#include #include namespace gs { @@ -205,24 +208,24 @@ std::string GraphPlannerWrapper::generate_jvm_options( physical::PhysicalPlan compilePlanJNI(jclass graph_planner_clz_, jmethodID graph_planner_method_id_, - jni::JNIEnvMark& jni_wrapper_, + JNIEnv* env, const std::string& compiler_config_path, const std::string& cypher_query_string) { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + nullptr); physical::PhysicalPlan physical_plan; if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { LOG(ERROR) << "Invalid GraphPlannerWrapper."; return physical_plan; } - jstring param1 = - jni_wrapper_.env()->NewStringUTF(compiler_config_path.c_str()); - jstring param2 = - jni_wrapper_.env()->NewStringUTF(cypher_query_string.c_str()); + jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); - jbyteArray res = (jbyteArray) jni_wrapper_.env()->CallStaticObjectMethod( + jbyteArray res = (jbyteArray) env->CallStaticObjectMethod( graph_planner_clz_, graph_planner_method_id_, param1, param2); - if (jni_wrapper_.env()->ExceptionCheck()) { - jni_wrapper_.env()->ExceptionDescribe(); - jni_wrapper_.env()->ExceptionClear(); + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); LOG(ERROR) << "Error in calling GraphPlanner."; return physical_plan; } @@ -230,15 +233,15 @@ physical::PhysicalPlan compilePlanJNI(jclass graph_planner_clz_, LOG(ERROR) << "Fail to generate plan."; return physical_plan; } - jbyte* str = jni_wrapper_.env()->GetByteArrayElements(res, NULL); - jsize len = jni_wrapper_.env()->GetArrayLength(res); + jbyte* str = env->GetByteArrayElements(res, NULL); + jsize len = env->GetArrayLength(res); LOG(INFO) << "Physical plan size: " << len; physical_plan.ParseFromArray(str, len); - jni_wrapper_.env()->ReleaseByteArrayElements(res, str, 0); - jni_wrapper_.env()->DeleteLocalRef(param1); - jni_wrapper_.env()->DeleteLocalRef(param2); - jni_wrapper_.env()->DeleteLocalRef(res); + env->ReleaseByteArrayElements(res, str, 0); + env->DeleteLocalRef(param1); + env->DeleteLocalRef(param2); + env->DeleteLocalRef(res); return physical_plan; } @@ -246,26 +249,37 @@ physical::PhysicalPlan compilePlanJNI(jclass graph_planner_clz_, #if (!GRAPH_PLANNER_JNI_INVOKER) -physical::PhysicalPlan readPhysicalPlan(const std::string& output_file) { - std::ifstream file(output_file, std::ios::binary); +void write_query_to_pipe(const std::string& path, + const std::string& query_str) { + LOG(INFO) << "write_query_to_pipe: " << path; - if (!file.is_open()) { - LOG(ERROR) << "Fail to open file: " << output_file; - return physical::PhysicalPlan(); + // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + int fd_to_java = open(path.c_str(), O_WRONLY); + if (fd_to_java < 0) { + LOG(ERROR) << "Fail to open pipe: " << path; + return; } + LOG(INFO) << "open pipe done"; + auto len = write(fd_to_java, query_str.c_str(), query_str.size()); + if (len != (int) query_str.size()) { + LOG(ERROR) << "Fail to write query to pipe:" << len; + return; + } + LOG(INFO) << "write_query_to_pipe done: " << len; + close(fd_to_java); +} - file.seekg(0, std::ios::end); - size_t size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::string buffer; - buffer.resize(size); - - file.read(&buffer[0], size); +void write_query_to_file(const std::string& path, + const std::string& query_str) { + std::ofstream query_file(path); + query_file << query_str; + query_file.close(); +} - file.close(); +physical::PhysicalPlan readPhysicalPlan(const std::string& plan_str) { + VLOG(10) << "plan str size: " << plan_str.size(); physical::PhysicalPlan plan; - if (!plan.ParseFromString(std::string(buffer))) { + if (!plan.ParseFromString(plan_str)) { LOG(ERROR) << "Fail to parse physical plan."; return physical::PhysicalPlan(); } @@ -278,17 +292,19 @@ physical::PhysicalPlan compilePlanSubprocess( const std::string& graph_statistic_json, const std::string& compiler_config_path, const std::string& cypher_query_string) { - // use execvp to run the command + physical::PhysicalPlan physical_plan; auto random_prefix = std::to_string( std::chrono::system_clock::now().time_since_epoch().count()); std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; - std::ofstream query_file(dst_query_path); - query_file << cypher_query_string; - query_file.close(); + VLOG(10) << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file; + mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + pid_t pid = fork(); + if (pid == 0) { - // generate a random file name for query const char* const command_string_array[] = { "java", "-cp", @@ -302,26 +318,40 @@ physical::PhysicalPlan compilePlanSubprocess( dst_output_file.c_str(), "/tmp/temp.cypher.yaml", NULL}; - LOG(INFO) << command_string_array[0] << " " << command_string_array[1] - << " " << command_string_array[2] << " " - << command_string_array[3] << " " << command_string_array[4] - << " " << command_string_array[5] << " " - << command_string_array[6] << " " << command_string_array[7] - << " " << command_string_array[8] << " " - << command_string_array[9]; execvp(command_string_array[0], const_cast(command_string_array)); } else if (pid < 0) { LOG(ERROR) << "Error in fork."; } else { + write_query_to_pipe(dst_query_path, cypher_query_string); + + int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); + if (fd_from_java < 0) { + LOG(ERROR) << "Fail to open pipe: " << dst_output_file; + return physical_plan; + } + std::vector stored_buffer; + char buffer[128]; + while (true) { + ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); + if (bytesRead <= 0) { + break; + } + stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); + } + physical_plan = readPhysicalPlan( + std::string(stored_buffer.begin(), stored_buffer.end())); + close(fd_from_java); + int status; waitpid(pid, &status, 0); if (status != 0) { LOG(ERROR) << "Error in running command."; } - return readPhysicalPlan(dst_output_file); } - return physical::PhysicalPlan(); + unlink(dst_query_path.c_str()); + unlink(dst_output_file.c_str()); + return physical_plan; } #endif @@ -330,7 +360,7 @@ physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( const std::string& cypher_query_string) { #if (GRAPH_PLANNER_JNI_INVOKER) return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, - jni_wrapper_, compiler_config_path, + jni_wrapper_.env(), compiler_config_path, cypher_query_string); #else return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, diff --git a/flex/planner/graph_planner.h b/flex/planner/graph_planner.h index 68e2325b61a2..d95a0792a50f 100644 --- a/flex/planner/graph_planner.h +++ b/flex/planner/graph_planner.h @@ -26,11 +26,9 @@ limitations under the License. #include "glog/logging.h" -// #define GRAPH_PLANNER_JNI_INVOKER JNI #ifndef GRAPH_PLANNER_JNI_INVOKER -#define GRAPH_PLANNER_JNI_INVOKER 0 // 1: JNI, 0: subprocess +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess #endif -// Could be JNI or subprocess namespace gs { diff --git a/flex/tests/planner/graph_planner_test.cc b/flex/tests/planner/graph_planner_test.cc index 46231d9150f0..58b680b6b804 100644 --- a/flex/tests/planner/graph_planner_test.cc +++ b/flex/tests/planner/graph_planner_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include int main(int argc, char** argv) { if (argc != 6) { @@ -40,11 +41,18 @@ int main(int argc, char** argv) { return 1; } gs::GraphPlannerWrapper planner(java_path, jna_path, graph_schema_path); - if (!planner.is_valid()) { - LOG(ERROR) << "Invalid GraphPlannerWrapper."; - return 1; + std::vector threads; + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + if (!planner.is_valid()) { + LOG(ERROR) << "Invalid GraphPlannerWrapper."; + } + auto plan = planner.CompilePlan(compiler_config_path, cypher_query); + CHECK(plan.plan_size() == 3) << "Invalid plan size: " << plan.plan_size(); + }); + } + for (auto& t : threads) { + t.join(); } - auto plan = planner.CompilePlan(compiler_config_path, cypher_query); - CHECK(plan.plan_size() == 3) << "Invalid plan size: " << plan.plan_size(); return 0; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 239592bd3b9b..b64265118d37 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -54,6 +54,14 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.RandomAccessFile; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -251,11 +259,23 @@ public static void main(String[] args) throws Exception { + " '' ''" + " 'optional '"); } - String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); + + BufferedReader reader = new BufferedReader(new FileReader(args[1])); + StringBuilder builder = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + builder.append(line); + } + String query = builder.toString(); + reader.close(); + Summary summary = generatePlan(args[0], query); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; - FileUtils.writeByteArrayToFile(new File(args[2]), physicalPlan.getContent()); + FileOutputStream fos = new FileOutputStream(args[2]); + fos.write(physicalPlan.getContent()); + fos.close(); + // write stored procedure meta to file LogicalPlan logicalPlan = summary.getLogicalPlan(); Configs extraConfigs = createExtraConfigs(args.length > 4 ? args[4] : null); From bc71bf60f977b63000a7d47929e295eb60718649 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 10 Dec 2024 21:43:37 +0800 Subject: [PATCH 06/27] introduce c++ wrapper for graph planner --- interactive_engine/compiler/build_native.xml | 60 +++ .../compiler/src/main/cpp/CMakeLists.txt | 5 + .../compiler/src/main/cpp/README.md | 3 + .../compiler/src/main/cpp/graph_planner.cc | 390 ++++++++++++++++++ .../compiler/src/main/cpp/graph_planner.h | 141 +++++++ .../meta/procedure/StoredProcedureMeta.java | 13 +- .../ir/meta/schema/GSDataTypeConvertor.java | 87 ++-- .../common/ir/meta/schema/Utils.java | 2 +- .../common/ir/tools/GraphPlanner.java | 28 +- .../alibaba/graphscope/common/ir/JNITest.java | 38 ++ interactive_engine/pom.xml | 36 ++ 11 files changed, 753 insertions(+), 50 deletions(-) create mode 100644 interactive_engine/compiler/build_native.xml create mode 100644 interactive_engine/compiler/src/main/cpp/CMakeLists.txt create mode 100644 interactive_engine/compiler/src/main/cpp/README.md create mode 100644 interactive_engine/compiler/src/main/cpp/graph_planner.cc create mode 100644 interactive_engine/compiler/src/main/cpp/graph_planner.h create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml new file mode 100644 index 000000000000..161f6d581a91 --- /dev/null +++ b/interactive_engine/compiler/build_native.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000000..ada8073abe65 --- /dev/null +++ b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE PLANNER_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") + +add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) +target_link_libraries(graph_planner PUBLIC flex_plan_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES}) +target_include_directories(graph_planner PUBLIC $) \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/cpp/README.md b/interactive_engine/compiler/src/main/cpp/README.md new file mode 100644 index 000000000000..cae277f0e1f3 --- /dev/null +++ b/interactive_engine/compiler/src/main/cpp/README.md @@ -0,0 +1,3 @@ +# GraphScope Interactive Query Planner + +This directory contains the c++ wrapper of Interactive Query Planner, which is implemented in [java](https://github.com/alibaba/GraphScope/tree/main/interactive_engine/compiler). \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.cc b/interactive_engine/compiler/src/main/cpp/graph_planner.cc new file mode 100644 index 000000000000..ba85c1cd69f8 --- /dev/null +++ b/interactive_engine/compiler/src/main/cpp/graph_planner.cc @@ -0,0 +1,390 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace gs { +#if (GRAPH_PLANNER_JNI_INVOKER) +namespace jni { + +static JavaVM* _jvm = NULL; + +JavaVM* CreateJavaVM(const std::string& jvm_options) { + const char *p, *q; + const char* jvm_opts; + if (jvm_options.empty()) { + jvm_opts = getenv("FLEX_JVM_OPTS"); + } else { + jvm_opts = jvm_options.c_str(); + } + if (jvm_opts == NULL) { + LOG(FATAL) << "Expect FLEX_JVM_OPTS set before initiate jvm"; + return NULL; + } + VLOG(1) << "Jvm opts str: " << jvm_opts; + + if (*jvm_opts == '\0') + return NULL; + + int num_of_opts = 1; + for (const char* p = jvm_opts; *p; p++) { + if (*p == ' ') + num_of_opts++; + } + + if (num_of_opts == 0) + return NULL; + + JavaVM* jvm = NULL; + JNIEnv* env = NULL; + int i = 0; + int status = 1; + JavaVMInitArgs vm_args; + + JavaVMOption* options = new JavaVMOption[num_of_opts]; + memset(options, 0, sizeof(JavaVMOption) * num_of_opts); + + for (p = q = jvm_opts;; p++) { + if (*p == ' ' || *p == '\0') { + if (q >= p) { + goto ret; + } + char* opt = new char[p - q + 1]; + memcpy(opt, q, p - q); + opt[p - q] = '\0'; + options[i++].optionString = opt; + q = p + 1; // assume opts are separated by single space + if (*p == '\0') + break; + } + } + + memset(&vm_args, 0, sizeof(vm_args)); + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = num_of_opts; + vm_args.options = options; + + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + if (status == JNI_OK) { + LOG(INFO) << "Create java virtual machine successfully."; + } else if (status == JNI_EEXIST) { + VLOG(1) << "JNI evn already exists."; + } else { + LOG(ERROR) << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")\n"; + } + +ret: + for (int i = 0; i < num_of_opts; i++) { + delete[] options[i].optionString; + } + delete[] options; + return jvm; +} + +// One process can only create jvm for once. +JavaVM* GetJavaVM(const std::string jvm_options = "") { + if (_jvm == NULL) { + // Try to find whether there exists one javaVM + jsize nVMs; + JNI_GetCreatedJavaVMs(NULL, 0, + &nVMs); // 1. just get the required array length + VLOG(1) << "Found " << nVMs << " VMs existing in this process."; + JavaVM** buffer = new JavaVM*[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + for (auto i = 0; i < nVMs; ++i) { + if (buffer[i] != NULL) { + _jvm = buffer[i]; + VLOG(1) << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm); + return _jvm; + } + } + _jvm = CreateJavaVM(jvm_options); + VLOG(1) << "Created JVM " << reinterpret_cast(_jvm); + } + return _jvm; +} + +JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + +JNIEnvMark::JNIEnvMark(const std::string& jvm_options) : _env(NULL) { + if (!GetJavaVM(jvm_options)) { + return; + } + int status = + GetJavaVM(jvm_options) + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + if (status != JNI_OK) { + LOG(ERROR) << "Error attach current thread: " << status; + } +} + +JNIEnvMark::~JNIEnvMark() { + if (_env) { + GetJavaVM()->DetachCurrentThread(); + } +} + +JNIEnv* JNIEnvMark::env() { return _env; } + +} // namespace jni + +#endif + +std::vector list_files(const std::string& path) { + // list all files in the directory + std::vector files; + for (const auto& entry : std::filesystem::directory_iterator(path)) { + files.push_back(entry.path().string()); + } + return files; +} + +std::string GraphPlannerWrapper::expand_directory(const std::string& path) { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) { + auto sub_path = path.substr(start, end - start); + if (!sub_path.empty()) { + if (std::filesystem::is_directory(sub_path)) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } else { + paths.push_back(sub_path); + } + } + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + if (!sub_path.empty()) { + auto files = list_files(sub_path); + paths.insert(paths.end(), files.begin(), files.end()); + } + std::stringstream ss; + for (const auto& p : paths) { + ss << p << ":"; + } + return ss.str(); +} + +#if (GRAPH_PLANNER_JNI_INVOKER) + +std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json) { + auto expanded_java_path = expand_directory(java_path); + VLOG(10) << "Expanded java path: " << expanded_java_path; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + if (!graph_statistic_json.empty()) { + jvm_options += " -Dgraph.statistic=" + graph_statistic_json; + } + return jvm_options; +} + +struct Plan { + physical::PhysicalPlan physical_plan; + std::string result_schema; +} + +Plan compilePlanJNI(jclass graph_planner_clz_, + jmethodID graph_planner_method_id_, + JNIEnv* env, + const std::string& compiler_config_path, + const std::string& cypher_query_string) { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + nullptr); + Plan plan; + if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { + LOG(ERROR) << "Invalid GraphPlannerWrapper."; + return plan; + } + jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + + // invoke jvm static function to get results as Object[] + jobjectArray resultArray = (jobjectArray) env->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2); + + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + LOG(ERROR) << "Error in calling GraphPlanner."; + return plan; + } + + // 0-th object is the physical plan in byte array + jbyteArray res = (jbyteArray)env->GetObjectArrayElement(resultArray, 0); + // 1-th object is the result schema in yaml format + jstring res2 = (jstring)env->GetObjectArrayElement(resultArray, 1); + + if (res == NULL || res2 == NULL) { + LOG(ERROR) << "Fail to generate plan."; + return plan; + } + jbyte* str = env->GetByteArrayElements(res, NULL); + jsize len = env->GetArrayLength(res); + LOG(INFO) << "Physical plan size: " << len; + + plan.physical_plan.ParseFromArray(str, len); + plan.result_schema = env->GetStringUTFChars(res2, NULL); + + env->ReleaseByteArrayElements(res, str, 0); + env->DeleteLocalRef(param1); + env->DeleteLocalRef(param2); + env->DeleteLocalRef(res); + // remove new added jni objects + env->DeleteLocalRef(res2); + env->DeleteLocalRef(resultArray); + + return physical_plan; +} +#endif + +#if (!GRAPH_PLANNER_JNI_INVOKER) + +void write_query_to_pipe(const std::string& path, + const std::string& query_str) { + LOG(INFO) << "write_query_to_pipe: " << path; + + // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + int fd_to_java = open(path.c_str(), O_WRONLY); + if (fd_to_java < 0) { + LOG(ERROR) << "Fail to open pipe: " << path; + return; + } + LOG(INFO) << "open pipe done"; + auto len = write(fd_to_java, query_str.c_str(), query_str.size()); + if (len != (int) query_str.size()) { + LOG(ERROR) << "Fail to write query to pipe:" << len; + return; + } + LOG(INFO) << "write_query_to_pipe done: " << len; + close(fd_to_java); +} + +void write_query_to_file(const std::string& path, + const std::string& query_str) { + std::ofstream query_file(path); + query_file << query_str; + query_file.close(); +} + +physical::PhysicalPlan readPhysicalPlan(const std::string& plan_str) { + VLOG(10) << "plan str size: " << plan_str.size(); + physical::PhysicalPlan plan; + if (!plan.ParseFromString(plan_str)) { + LOG(ERROR) << "Fail to parse physical plan."; + return physical::PhysicalPlan(); + } + return plan; +} + +physical::PhysicalPlan compilePlanSubprocess( + const std::string& class_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json, + const std::string& compiler_config_path, + const std::string& cypher_query_string) { + physical::PhysicalPlan physical_plan; + auto random_prefix = std::to_string( + std::chrono::system_clock::now().time_since_epoch().count()); + std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; + std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; + VLOG(10) << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file; + mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + + pid_t pid = fork(); + + if (pid == 0) { + const char* const command_string_array[] = { + "java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + "com.alibaba.graphscope.common.ir.tools.GraphPlanner", + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; + execvp(command_string_array[0], + const_cast(command_string_array)); + } else if (pid < 0) { + LOG(ERROR) << "Error in fork."; + } else { + write_query_to_pipe(dst_query_path, cypher_query_string); + + int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); + if (fd_from_java < 0) { + LOG(ERROR) << "Fail to open pipe: " << dst_output_file; + return physical_plan; + } + std::vector stored_buffer; + char buffer[128]; + while (true) { + ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); + if (bytesRead <= 0) { + break; + } + stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); + } + physical_plan = readPhysicalPlan( + std::string(stored_buffer.begin(), stored_buffer.end())); + close(fd_from_java); + + int status; + waitpid(pid, &status, 0); + if (status != 0) { + LOG(ERROR) << "Error in running command."; + } + } + unlink(dst_query_path.c_str()); + unlink(dst_output_file.c_str()); + return physical_plan; +} +#endif + +physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( + const std::string& compiler_config_path, + const std::string& cypher_query_string) { +#if (GRAPH_PLANNER_JNI_INVOKER) + return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, + jni_wrapper_.env(), compiler_config_path, + cypher_query_string); +#else + return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, + graph_statistic_json_, compiler_config_path, + cypher_query_string); +#endif +} + +} // namespace gs diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.h b/interactive_engine/compiler/src/main/cpp/graph_planner.h new file mode 100644 index 000000000000..d95a0792a50f --- /dev/null +++ b/interactive_engine/compiler/src/main/cpp/graph_planner.h @@ -0,0 +1,141 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#ifndef PLANNER_GRAPH_PLANNER_H_ +#define PLANNER_GRAPH_PLANNER_H_ + +#include +#include +#include +#include +#include + +#include "flex/proto_generated_gie/physical.pb.h" + +#include "glog/logging.h" + +#ifndef GRAPH_PLANNER_JNI_INVOKER +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess +#endif + +namespace gs { + +#if (GRAPH_PLANNER_JNI_INVOKER) +namespace jni { +struct JNIEnvMark { + JNIEnv* _env; + + JNIEnvMark(); + JNIEnvMark(const std::string& jvm_options); + ~JNIEnvMark(); + JNIEnv* env(); +}; + +} // namespace jni +#endif + +class GraphPlannerWrapper { + public: + static constexpr const char* kGraphPlannerClass = + "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; + static constexpr const char* kGraphPlannerMethod = "generatePhysicalPlan"; + static constexpr const char* kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;)[B"; + + GraphPlannerWrapper(const std::string java_path, const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json = "") +#if (GRAPH_PLANNER_JNI_INVOKER) + : jni_wrapper_(generate_jvm_options( + java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { + jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); + if (clz == NULL) { + LOG(ERROR) << "Fail to find class: " << kGraphPlannerClass; + return; + } + graph_planner_clz_ = (jclass) jni_wrapper_.env()->NewGlobalRef(clz); + jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( + graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); + if (j_method_id == NULL) { + LOG(ERROR) << "Fail to find method: " << kGraphPlannerMethod; + return; + } + graph_planner_method_id_ = j_method_id; + } +#else + : jna_path_("-Djna.library.path=" + jna_path), + graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), + graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { + class_path_ = expand_directory(java_path); + } +#endif + + ~GraphPlannerWrapper() { +#if (GRAPH_PLANNER_JNI_INVOKER) + if (graph_planner_clz_ != NULL) { + jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); + } +#endif + } + + inline bool is_valid() { +#if (GRAPH_PLANNER_JNI_INVOKER) + return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; +#else + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. +#endif + } + + /** + * @brief Invoker GraphPlanner to generate a physical plan from a cypher + * query. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return physical plan in string. + */ + physical::PhysicalPlan CompilePlan(const std::string& compiler_config_path, + const std::string& cypher_query_string); + + private: + std::string generate_jvm_options(const std::string java_path, + const std::string& jna_path, + const std::string& graph_schema_yaml, + const std::string& graph_statistic_json); + // physical::PhysicalPlan compilePlanJNI(const std::string& + // compiler_config_path, + // const std::string& + // cypher_query_string); + std::string expand_directory(const std::string& path); +#if (GRAPH_PLANNER_JNI_INVOKER) + // We need to list all files in the directory, if exists. + // The reason why we need to list all files in the directory is that + // java -Djava.class.path=dir/* (in jni, which we are using)will not load all + // jar files in the directory, While java -cp dir/* will load all jar files in + // the directory. + + gs::jni::JNIEnvMark jni_wrapper_; + jclass graph_planner_clz_; + jmethodID graph_planner_method_id_; +#else + std::string class_path_; + std::string jna_path_; + std::string graph_schema_yaml_; + std::string graph_statistic_json_; +#endif +}; +} // namespace gs + +#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index c0e00ddde078..15b16dcbcf71 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -163,16 +163,19 @@ public int hashCode() { } public static class Serializer { - public static void perform(StoredProcedureMeta meta, OutputStream outputStream) + public static void perform( + StoredProcedureMeta meta, OutputStream outputStream, boolean throwsOnFail) throws IOException { Yaml yaml = new Yaml(); - String mapStr = yaml.dump(createProduceMetaMap(meta)); + String mapStr = yaml.dump(createProduceMetaMap(meta, throwsOnFail)); outputStream.write(mapStr.getBytes(StandardCharsets.UTF_8)); } - private static Map createProduceMetaMap(StoredProcedureMeta meta) { + private static Map createProduceMetaMap( + StoredProcedureMeta meta, boolean throwsOnFail) { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + GSDataTypeConvertor.Factory.create( + RelDataType.class, typeFactory, throwsOnFail); return ImmutableMap.of( Config.NAME.getKey(), meta.name, @@ -217,7 +220,7 @@ private static Map createProduceMetaMap(StoredProcedureMeta meta public static class Deserializer { public static StoredProcedureMeta perform(InputStream inputStream) throws IOException { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory, true); Yaml yaml = new Yaml(); Map config = yaml.load(inputStream); return new StoredProcedureMeta( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java index f12a70e8caf8..8eaa8898d032 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java @@ -38,7 +38,8 @@ public interface GSDataTypeConvertor { GSDataTypeDesc convert(T from); class Factory { - public static GSDataTypeConvertor create(Class tType, @Nullable Object config) { + public static GSDataTypeConvertor create( + Class tType, @Nullable Object config, boolean throwsOnFail) { if (tType.equals(DataType.class)) { return new GSDataTypeConvertor() { @Override @@ -58,20 +59,24 @@ public DataType convert(GSDataTypeDesc from) { case "DT_DOUBLE": return DataType.DOUBLE; default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } } } else if ((value = typeMap.get("string")) != null) { Map strType = (Map) value; if (strType.containsKey("long_text")) { return DataType.STRING; } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } } } else if ((value = typeMap.get("temporal")) != null) { Map temporalType = (Map) value; @@ -82,15 +87,22 @@ public DataType convert(GSDataTypeDesc from) { } else if (temporalType.containsKey("timestamp")) { return DataType.TIMESTAMP; } else { + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } + } + } else { + if (throwsOnFail) { throw new UnsupportedOperationException( "can not convert GSDataTypeDesc [" + from + "] to DataType"); } - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); } + return DataType.UNKNOWN; } @Override @@ -121,20 +133,24 @@ public RelDataType convert(GSDataTypeDesc from) { case "DT_DOUBLE": return typeFactory.createSqlType(SqlTypeName.DOUBLE); default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("string")) != null) { Map strType = (Map) value; if (strType.containsKey("long_text")) { return typeFactory.createSqlType(SqlTypeName.CHAR); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("temporal")) != null) { Map temporalType = (Map) value; @@ -145,10 +161,12 @@ public RelDataType convert(GSDataTypeDesc from) { } else if (temporalType.containsKey("timestamp")) { return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("array")) != null) { Map arrayType = (Map) value; @@ -174,9 +192,14 @@ public RelDataType convert(GSDataTypeDesc from) { convert(new GSDataTypeDesc(keyType)), convert(new GSDataTypeDesc(valueType))); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } + return typeFactory.createUnknownType(); } @Override @@ -256,10 +279,14 @@ public GSDataTypeDesc convert(RelDataType from) { valueType)); break; default: - throw new UnsupportedOperationException( - "can not convert RelDataType [" - + from - + "] to GSDataTypeDesc"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert RelDataType [" + + from + + "] to GSDataTypeDesc"); + } else { + yamlDesc = ImmutableMap.of("primitive_type", "DT_UNKNOWN"); + } } return new GSDataTypeDesc(yamlDesc); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java index 0b59ccf7d216..fef0aad52841 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java @@ -52,7 +52,7 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(DataType.class, null); + GSDataTypeConvertor.Factory.create(DataType.class, null, true); builderGraphElementFromYaml( (List) Objects.requireNonNull( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index b64265118d37..b346cfe0d744 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -52,17 +52,7 @@ import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import java.io.File; -import java.io.FileOutputStream; -import java.io.RandomAccessFile; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -241,11 +231,21 @@ public static Summary generatePlan(String config_path, String query_string) thro return summary; } - public static byte[] generatePhysicalPlan(String config_path, String query_string) + public static Object[] generatePhysicalPlan(String config_path, String query_string) throws Exception { Summary summary = generatePlan(config_path, query_string); + LogicalPlan logicalPlan = summary.getLogicalPlan(); PhysicalPlan physicalPlan = summary.physicalPlan; - return physicalPlan.getContent(); + Configs extraConfigs = createExtraConfigs(null); + StoredProcedureMeta procedureMeta = + new StoredProcedureMeta( + extraConfigs, + query_string, + logicalPlan.getOutputType(), + logicalPlan.getDynamicParams()); + ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); + StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); + return new Object[] {physicalPlan.getContent(), new String(metaStream.toByteArray())}; } public static void main(String[] args) throws Exception { @@ -285,6 +285,6 @@ public static void main(String[] args) throws Exception { query, logicalPlan.getOutputType(), logicalPlan.getDynamicParams()); - StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3])); + StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3]), true); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java new file mode 100644 index 000000000000..5469e8ad55bd --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java @@ -0,0 +1,38 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir; + +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; + +import org.junit.Test; + +public class JNITest { + @Test + public void test() throws Exception { + Object[] objects = + GraphPlanner.generatePhysicalPlan( + "conf/ir.compiler.properties", "Match (n) Return n, count(n)"); + GraphAlgebraPhysical.PhysicalPlan plan = + GraphAlgebraPhysical.PhysicalPlan.parseFrom((byte[]) objects[0]); + System.out.println(plan); + String resultSchema = (String) objects[1]; + System.out.println(resultSchema); + } +} diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index f69d68faca14..ebe5c8063194 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -49,6 +49,36 @@ compiler + + graph-planner-jni + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + + run + + compile + + + + + + + + + + + + + graphscope @@ -698,6 +728,12 @@ ${interactive.sdk.version} ${interactive.sdk.classifier} + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + maven-plugin + From 93b43fa3905aa4a91d1fa3d1c3298cb99d50bde5 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 11 Dec 2024 12:13:18 +0800 Subject: [PATCH 07/27] fix compilation Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 8 +- flex/CMakeLists.txt | 10 - flex/planner/CMakeLists.txt | 5 - flex/planner/README.md | 3 - flex/planner/graph_planner.cc | 372 ------------------ flex/planner/graph_planner.h | 141 ------- flex/tests/CMakeLists.txt | 3 +- flex/tests/planner/CMakeLists.txt | 2 - flex/tests/planner/graph_planner_test.cc | 58 --- interactive_engine/compiler/build_native.xml | 19 +- interactive_engine/compiler/pom.xml | 30 ++ .../compiler/src/main/cpp/CMakeLists.txt | 72 +++- .../compiler/src/main/cpp/graph_planner.cc | 218 +++++----- .../compiler/src/main/cpp/graph_planner.h | 67 ++-- .../compiler/src/main/cpp/test/test.cc | 83 ++++ interactive_engine/pom.xml | 41 +- 16 files changed, 341 insertions(+), 791 deletions(-) delete mode 100644 flex/planner/CMakeLists.txt delete mode 100644 flex/planner/README.md delete mode 100644 flex/planner/graph_planner.cc delete mode 100644 flex/planner/graph_planner.h delete mode 100644 flex/tests/planner/CMakeLists.txt delete mode 100644 flex/tests/planner/graph_planner_test.cc create mode 100644 interactive_engine/compiler/src/main/cpp/test/test.cc diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 345f1f61bbe5..bd7c3ddb178e 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -319,7 +319,7 @@ jobs: - name: Test physical plan generation run: | - cd ${GITHUB_WORKSPACE}/flex/build + cd ${GITHUB_WORKSPACE}/interactive_engine/compiler cat > /tmp/physical_plan_gen_config.yaml <> /tmp/physical_plan_gen_config.yaml echo " meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml - - ./tests/planner/graph_planner_test ${GITHUB_WORKSPACE}/interactive_engine/compiler/target/compiler-0.0.1-SNAPSHOT.jar:${GITHUB_WORKSPACE}/interactive_engine/compiler/target/libs/ \ - ${GITHUB_WORKSPACE}/interactive_engine/executor/ir/target/release/ ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml \ - /tmp/physical_plan_gen_config.yaml "MATCH(n) return count(n);" + mvn clean install -DskipTests -Pgraph-planner-jni + ./target/native/test_graph_planner - name: Run End-to-End cypher adhoc ldbc query test env: diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 4439aece78e5..c0385658cec2 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -93,15 +93,6 @@ find_package(libgrapelite REQUIRED) include_directories(SYSTEM ${LIBGRAPELITE_INCLUDE_DIRS}) include_directories(SYSTEM ${LIBGRAPELITE_INCLUDE_DIRS}/grape/analytical_apps) -# find JNI----------------------------------------------------------------------- -find_package(JNI REQUIRED) -if (JNI_FOUND) - include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) - include_directories(SYSTEM ${JAVA_INCLUDE_PATH2}) -else() - message(FATAL_ERROR "JNI not found") -endif() - # find Threads------------------------------------------------------------------ set(CMAKE_THREAD_PREFER_PTHREAD ON) find_package(Threads REQUIRED) @@ -242,7 +233,6 @@ endif() if (BUILD_TEST) add_subdirectory(tests) endif() -add_subdirectory(planner) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flex-config.in.cmake diff --git a/flex/planner/CMakeLists.txt b/flex/planner/CMakeLists.txt deleted file mode 100644 index ada8073abe65..000000000000 --- a/flex/planner/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -file(GLOB_RECURSE PLANNER_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") - -add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) -target_link_libraries(graph_planner PUBLIC flex_plan_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES}) -target_include_directories(graph_planner PUBLIC $) \ No newline at end of file diff --git a/flex/planner/README.md b/flex/planner/README.md deleted file mode 100644 index cae277f0e1f3..000000000000 --- a/flex/planner/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# GraphScope Interactive Query Planner - -This directory contains the c++ wrapper of Interactive Query Planner, which is implemented in [java](https://github.com/alibaba/GraphScope/tree/main/interactive_engine/compiler). \ No newline at end of file diff --git a/flex/planner/graph_planner.cc b/flex/planner/graph_planner.cc deleted file mode 100644 index 06913a1b76aa..000000000000 --- a/flex/planner/graph_planner.cc +++ /dev/null @@ -1,372 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace gs { -#if (GRAPH_PLANNER_JNI_INVOKER) -namespace jni { - -static JavaVM* _jvm = NULL; - -JavaVM* CreateJavaVM(const std::string& jvm_options) { - const char *p, *q; - const char* jvm_opts; - if (jvm_options.empty()) { - jvm_opts = getenv("FLEX_JVM_OPTS"); - } else { - jvm_opts = jvm_options.c_str(); - } - if (jvm_opts == NULL) { - LOG(FATAL) << "Expect FLEX_JVM_OPTS set before initiate jvm"; - return NULL; - } - VLOG(1) << "Jvm opts str: " << jvm_opts; - - if (*jvm_opts == '\0') - return NULL; - - int num_of_opts = 1; - for (const char* p = jvm_opts; *p; p++) { - if (*p == ' ') - num_of_opts++; - } - - if (num_of_opts == 0) - return NULL; - - JavaVM* jvm = NULL; - JNIEnv* env = NULL; - int i = 0; - int status = 1; - JavaVMInitArgs vm_args; - - JavaVMOption* options = new JavaVMOption[num_of_opts]; - memset(options, 0, sizeof(JavaVMOption) * num_of_opts); - - for (p = q = jvm_opts;; p++) { - if (*p == ' ' || *p == '\0') { - if (q >= p) { - goto ret; - } - char* opt = new char[p - q + 1]; - memcpy(opt, q, p - q); - opt[p - q] = '\0'; - options[i++].optionString = opt; - q = p + 1; // assume opts are separated by single space - if (*p == '\0') - break; - } - } - - memset(&vm_args, 0, sizeof(vm_args)); - vm_args.version = JNI_VERSION_1_8; - vm_args.nOptions = num_of_opts; - vm_args.options = options; - - status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); - if (status == JNI_OK) { - LOG(INFO) << "Create java virtual machine successfully."; - } else if (status == JNI_EEXIST) { - VLOG(1) << "JNI evn already exists."; - } else { - LOG(ERROR) << "Error, create java virtual machine failed. return JNI_CODE (" - << status << ")\n"; - } - -ret: - for (int i = 0; i < num_of_opts; i++) { - delete[] options[i].optionString; - } - delete[] options; - return jvm; -} - -// One process can only create jvm for once. -JavaVM* GetJavaVM(const std::string jvm_options = "") { - if (_jvm == NULL) { - // Try to find whether there exists one javaVM - jsize nVMs; - JNI_GetCreatedJavaVMs(NULL, 0, - &nVMs); // 1. just get the required array length - VLOG(1) << "Found " << nVMs << " VMs existing in this process."; - JavaVM** buffer = new JavaVM*[nVMs]; - JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data - for (auto i = 0; i < nVMs; ++i) { - if (buffer[i] != NULL) { - _jvm = buffer[i]; - VLOG(1) << "Found index " << i << " VM non null " - << reinterpret_cast(_jvm); - return _jvm; - } - } - _jvm = CreateJavaVM(jvm_options); - VLOG(1) << "Created JVM " << reinterpret_cast(_jvm); - } - return _jvm; -} - -JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} - -JNIEnvMark::JNIEnvMark(const std::string& jvm_options) : _env(NULL) { - if (!GetJavaVM(jvm_options)) { - return; - } - int status = - GetJavaVM(jvm_options) - ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); - if (status != JNI_OK) { - LOG(ERROR) << "Error attach current thread: " << status; - } -} - -JNIEnvMark::~JNIEnvMark() { - if (_env) { - GetJavaVM()->DetachCurrentThread(); - } -} - -JNIEnv* JNIEnvMark::env() { return _env; } - -} // namespace jni - -#endif - -std::vector list_files(const std::string& path) { - // list all files in the directory - std::vector files; - for (const auto& entry : std::filesystem::directory_iterator(path)) { - files.push_back(entry.path().string()); - } - return files; -} - -std::string GraphPlannerWrapper::expand_directory(const std::string& path) { - std::vector paths; - std::string::size_type start = 0; - std::string::size_type end = path.find(':'); - while (end != std::string::npos) { - auto sub_path = path.substr(start, end - start); - if (!sub_path.empty()) { - if (std::filesystem::is_directory(sub_path)) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } else { - paths.push_back(sub_path); - } - } - start = end + 1; - end = path.find(':', start); - } - auto sub_path = path.substr(start); - if (!sub_path.empty()) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } - std::stringstream ss; - for (const auto& p : paths) { - ss << p << ":"; - } - return ss.str(); -} - -#if (GRAPH_PLANNER_JNI_INVOKER) - -std::string GraphPlannerWrapper::generate_jvm_options( - const std::string java_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json) { - auto expanded_java_path = expand_directory(java_path); - VLOG(10) << "Expanded java path: " << expanded_java_path; - std::string jvm_options = "-Djava.class.path=" + expanded_java_path; - jvm_options += " -Djna.library.path=" + jna_path; - jvm_options += " -Dgraph.schema=" + graph_schema_yaml; - if (!graph_statistic_json.empty()) { - jvm_options += " -Dgraph.statistic=" + graph_statistic_json; - } - return jvm_options; -} - -physical::PhysicalPlan compilePlanJNI(jclass graph_planner_clz_, - jmethodID graph_planner_method_id_, - JNIEnv* env, - const std::string& compiler_config_path, - const std::string& cypher_query_string) { - jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), - nullptr); - physical::PhysicalPlan physical_plan; - if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { - LOG(ERROR) << "Invalid GraphPlannerWrapper."; - return physical_plan; - } - jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); - jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); - - jbyteArray res = (jbyteArray) env->CallStaticObjectMethod( - graph_planner_clz_, graph_planner_method_id_, param1, param2); - if (env->ExceptionCheck()) { - env->ExceptionDescribe(); - env->ExceptionClear(); - LOG(ERROR) << "Error in calling GraphPlanner."; - return physical_plan; - } - if (res == NULL) { - LOG(ERROR) << "Fail to generate plan."; - return physical_plan; - } - jbyte* str = env->GetByteArrayElements(res, NULL); - jsize len = env->GetArrayLength(res); - LOG(INFO) << "Physical plan size: " << len; - - physical_plan.ParseFromArray(str, len); - env->ReleaseByteArrayElements(res, str, 0); - env->DeleteLocalRef(param1); - env->DeleteLocalRef(param2); - env->DeleteLocalRef(res); - - return physical_plan; -} -#endif - -#if (!GRAPH_PLANNER_JNI_INVOKER) - -void write_query_to_pipe(const std::string& path, - const std::string& query_str) { - LOG(INFO) << "write_query_to_pipe: " << path; - - // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - int fd_to_java = open(path.c_str(), O_WRONLY); - if (fd_to_java < 0) { - LOG(ERROR) << "Fail to open pipe: " << path; - return; - } - LOG(INFO) << "open pipe done"; - auto len = write(fd_to_java, query_str.c_str(), query_str.size()); - if (len != (int) query_str.size()) { - LOG(ERROR) << "Fail to write query to pipe:" << len; - return; - } - LOG(INFO) << "write_query_to_pipe done: " << len; - close(fd_to_java); -} - -void write_query_to_file(const std::string& path, - const std::string& query_str) { - std::ofstream query_file(path); - query_file << query_str; - query_file.close(); -} - -physical::PhysicalPlan readPhysicalPlan(const std::string& plan_str) { - VLOG(10) << "plan str size: " << plan_str.size(); - physical::PhysicalPlan plan; - if (!plan.ParseFromString(plan_str)) { - LOG(ERROR) << "Fail to parse physical plan."; - return physical::PhysicalPlan(); - } - return plan; -} - -physical::PhysicalPlan compilePlanSubprocess( - const std::string& class_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json, - const std::string& compiler_config_path, - const std::string& cypher_query_string) { - physical::PhysicalPlan physical_plan; - auto random_prefix = std::to_string( - std::chrono::system_clock::now().time_since_epoch().count()); - std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; - std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; - VLOG(10) << "dst_query_path: " << dst_query_path - << " dst_output_file: " << dst_output_file; - mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - - pid_t pid = fork(); - - if (pid == 0) { - const char* const command_string_array[] = { - "java", - "-cp", - class_path.c_str(), - jna_path.c_str(), - graph_schema_yaml.c_str(), - graph_statistic_json.c_str(), - "com.alibaba.graphscope.common.ir.tools.GraphPlanner", - compiler_config_path.c_str(), - dst_query_path.c_str(), - dst_output_file.c_str(), - "/tmp/temp.cypher.yaml", - NULL}; - execvp(command_string_array[0], - const_cast(command_string_array)); - } else if (pid < 0) { - LOG(ERROR) << "Error in fork."; - } else { - write_query_to_pipe(dst_query_path, cypher_query_string); - - int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); - if (fd_from_java < 0) { - LOG(ERROR) << "Fail to open pipe: " << dst_output_file; - return physical_plan; - } - std::vector stored_buffer; - char buffer[128]; - while (true) { - ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); - if (bytesRead <= 0) { - break; - } - stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); - } - physical_plan = readPhysicalPlan( - std::string(stored_buffer.begin(), stored_buffer.end())); - close(fd_from_java); - - int status; - waitpid(pid, &status, 0); - if (status != 0) { - LOG(ERROR) << "Error in running command."; - } - } - unlink(dst_query_path.c_str()); - unlink(dst_output_file.c_str()); - return physical_plan; -} -#endif - -physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( - const std::string& compiler_config_path, - const std::string& cypher_query_string) { -#if (GRAPH_PLANNER_JNI_INVOKER) - return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, - jni_wrapper_.env(), compiler_config_path, - cypher_query_string); -#else - return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, - graph_statistic_json_, compiler_config_path, - cypher_query_string); -#endif -} - -} // namespace gs diff --git a/flex/planner/graph_planner.h b/flex/planner/graph_planner.h deleted file mode 100644 index d95a0792a50f..000000000000 --- a/flex/planner/graph_planner.h +++ /dev/null @@ -1,141 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#ifndef PLANNER_GRAPH_PLANNER_H_ -#define PLANNER_GRAPH_PLANNER_H_ - -#include -#include -#include -#include -#include - -#include "flex/proto_generated_gie/physical.pb.h" - -#include "glog/logging.h" - -#ifndef GRAPH_PLANNER_JNI_INVOKER -#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess -#endif - -namespace gs { - -#if (GRAPH_PLANNER_JNI_INVOKER) -namespace jni { -struct JNIEnvMark { - JNIEnv* _env; - - JNIEnvMark(); - JNIEnvMark(const std::string& jvm_options); - ~JNIEnvMark(); - JNIEnv* env(); -}; - -} // namespace jni -#endif - -class GraphPlannerWrapper { - public: - static constexpr const char* kGraphPlannerClass = - "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; - static constexpr const char* kGraphPlannerMethod = "generatePhysicalPlan"; - static constexpr const char* kGraphPlannerMethodSignature = - "(Ljava/lang/String;Ljava/lang/String;)[B"; - - GraphPlannerWrapper(const std::string java_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json = "") -#if (GRAPH_PLANNER_JNI_INVOKER) - : jni_wrapper_(generate_jvm_options( - java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { - jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); - if (clz == NULL) { - LOG(ERROR) << "Fail to find class: " << kGraphPlannerClass; - return; - } - graph_planner_clz_ = (jclass) jni_wrapper_.env()->NewGlobalRef(clz); - jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( - graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); - if (j_method_id == NULL) { - LOG(ERROR) << "Fail to find method: " << kGraphPlannerMethod; - return; - } - graph_planner_method_id_ = j_method_id; - } -#else - : jna_path_("-Djna.library.path=" + jna_path), - graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), - graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { - class_path_ = expand_directory(java_path); - } -#endif - - ~GraphPlannerWrapper() { -#if (GRAPH_PLANNER_JNI_INVOKER) - if (graph_planner_clz_ != NULL) { - jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); - } -#endif - } - - inline bool is_valid() { -#if (GRAPH_PLANNER_JNI_INVOKER) - return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; -#else - return true; // just return true, since we don't have a way to check the - // validity when calling via subprocess. -#endif - } - - /** - * @brief Invoker GraphPlanner to generate a physical plan from a cypher - * query. - * @param compiler_config_path The path of compiler config file. - * @param cypher_query_string The cypher query string. - * @return physical plan in string. - */ - physical::PhysicalPlan CompilePlan(const std::string& compiler_config_path, - const std::string& cypher_query_string); - - private: - std::string generate_jvm_options(const std::string java_path, - const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json); - // physical::PhysicalPlan compilePlanJNI(const std::string& - // compiler_config_path, - // const std::string& - // cypher_query_string); - std::string expand_directory(const std::string& path); -#if (GRAPH_PLANNER_JNI_INVOKER) - // We need to list all files in the directory, if exists. - // The reason why we need to list all files in the directory is that - // java -Djava.class.path=dir/* (in jni, which we are using)will not load all - // jar files in the directory, While java -cp dir/* will load all jar files in - // the directory. - - gs::jni::JNIEnvMark jni_wrapper_; - jclass graph_planner_clz_; - jmethodID graph_planner_method_id_; -#else - std::string class_path_; - std::string jna_path_; - std::string graph_schema_yaml_; - std::string graph_statistic_json_; -#endif -}; -} // namespace gs - -#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file diff --git a/flex/tests/CMakeLists.txt b/flex/tests/CMakeLists.txt index d09a89e0ce9f..4c851c54b9a4 100644 --- a/flex/tests/CMakeLists.txt +++ b/flex/tests/CMakeLists.txt @@ -1,3 +1,2 @@ add_subdirectory(hqps) -add_subdirectory(rt_mutable_graph) -add_subdirectory(planner) \ No newline at end of file +add_subdirectory(rt_mutable_graph) \ No newline at end of file diff --git a/flex/tests/planner/CMakeLists.txt b/flex/tests/planner/CMakeLists.txt deleted file mode 100644 index 0ca1c10a4884..000000000000 --- a/flex/tests/planner/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable(graph_planner_test graph_planner_test.cc) -target_link_libraries(graph_planner_test graph_planner ${GLOG_LIBRARIES}) \ No newline at end of file diff --git a/flex/tests/planner/graph_planner_test.cc b/flex/tests/planner/graph_planner_test.cc deleted file mode 100644 index 58b680b6b804..000000000000 --- a/flex/tests/planner/graph_planner_test.cc +++ /dev/null @@ -1,58 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "flex/planner/graph_planner.h" -#include -#include -#include -#include - -int main(int argc, char** argv) { - if (argc != 6) { - LOG(ERROR) << "Usage: " << argv[0] - << " " - " "; - LOG(ERROR) << "But got " << argc << " arguments."; - return 1; - } - std::string java_path = argv[1]; - std::string jna_path = argv[2]; - std::string graph_schema_path = argv[3]; - std::string compiler_config_path = argv[4]; - std::string cypher_query = argv[5]; - if (java_path.empty() || jna_path.empty() || graph_schema_path.empty()) { - LOG(ERROR) << "Invalid input."; - return 1; - } - if (!std::filesystem::exists(compiler_config_path)) { - LOG(ERROR) << "Invalid compiler config path."; - return 1; - } - gs::GraphPlannerWrapper planner(java_path, jna_path, graph_schema_path); - std::vector threads; - for (int i = 0; i < 10; ++i) { - threads.emplace_back([&]() { - if (!planner.is_valid()) { - LOG(ERROR) << "Invalid GraphPlannerWrapper."; - } - auto plan = planner.CompilePlan(compiler_config_path, cypher_query); - CHECK(plan.plan_size() == 3) << "Invalid plan size: " << plan.plan_size(); - }); - } - for (auto& t : threads) { - t.join(); - } - return 0; -} diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml index 161f6d581a91..049909547f46 100644 --- a/interactive_engine/compiler/build_native.xml +++ b/interactive_engine/compiler/build_native.xml @@ -13,8 +13,8 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + + @@ -34,19 +34,14 @@ - + - + - - - - - - + + @@ -54,7 +49,7 @@ - + diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index 76f014bb17f3..df04307d853c 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -224,6 +224,36 @@ + + graph-planner-jni + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + + run + + compile + + + + + + + + + + + + + diff --git a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt index ada8073abe65..53bc20f0f9fa 100644 --- a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt +++ b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt @@ -1,5 +1,71 @@ -file(GLOB_RECURSE PLANNER_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") +# Copyright 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.1) +project(COMPILER-JNI + LANGUAGES CXX + VERSION 0.0.1) +option(BUILD_TEST "Whether to build test" ON) + +set(JAVA_AWT_INCLUDE_PATH NotNeeded) +find_package(JNI QUIET) +if (JNI_FOUND) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH2}) +else() + message(FATAL_ERROR "JNI not found") +endif() + +find_package(Protobuf REQUIRED) +include_directories(${Protobuf_INCLUDE_DIRS}) + +set(GIE_COMPILER_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../executor/ir/proto/) +# proto files of gie compiler +set(CODE_GEN_PROTOBUF_FILES + ${GIE_COMPILER_PROTO_DIR}/algebra.proto + ${GIE_COMPILER_PROTO_DIR}/common.proto + ${GIE_COMPILER_PROTO_DIR}/expr.proto + ${GIE_COMPILER_PROTO_DIR}/physical.proto + ${GIE_COMPILER_PROTO_DIR}/results.proto + ${GIE_COMPILER_PROTO_DIR}/schema.proto + ${GIE_COMPILER_PROTO_DIR}/type.proto + ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto +) + +#create directory first +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie) + +# proto gen for gie proto +protobuf_generate(APPEND_PATH + TARGET ${LOCAL_EXE_NAME} + LANGUAGE cpp + OUT_VAR PROTO_SRCS_GIE + PROTOS ${CODE_GEN_PROTOBUF_FILES} + IMPORT_DIRS ${GIE_COMPILER_PROTO_DIR} + PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie +) + +add_library(ir_proto SHARED ${PROTO_SRCS_GIE}) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +file(GLOB_RECURSE PLANNER_SRC_FILES "graph_planner.cc") add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) -target_link_libraries(graph_planner PUBLIC flex_plan_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES}) -target_include_directories(graph_planner PUBLIC $) \ No newline at end of file +target_link_libraries(graph_planner PUBLIC ir_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES} ${Protobuf_LIBRARIES}) +target_include_directories(graph_planner PUBLIC $) + +if (BUILD_TEST) + add_executable(test_graph_planner test/test.cc) + target_link_libraries(test_graph_planner PRIVATE graph_planner) +endif() \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.cc b/interactive_engine/compiler/src/main/cpp/graph_planner.cc index ba85c1cd69f8..417169a1301b 100644 --- a/interactive_engine/compiler/src/main/cpp/graph_planner.cc +++ b/interactive_engine/compiler/src/main/cpp/graph_planner.cc @@ -12,41 +12,41 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include #include #include #include #include -#include +#include "graph_planner.h" #include -#include namespace gs { #if (GRAPH_PLANNER_JNI_INVOKER) namespace jni { -static JavaVM* _jvm = NULL; +static JavaVM *_jvm = NULL; -JavaVM* CreateJavaVM(const std::string& jvm_options) { +JavaVM *CreateJavaVM(const std::string &jvm_options) { const char *p, *q; - const char* jvm_opts; + const char *jvm_opts; if (jvm_options.empty()) { jvm_opts = getenv("FLEX_JVM_OPTS"); } else { jvm_opts = jvm_options.c_str(); } if (jvm_opts == NULL) { - LOG(FATAL) << "Expect FLEX_JVM_OPTS set before initiate jvm"; + std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; return NULL; } - VLOG(1) << "Jvm opts str: " << jvm_opts; + std::cout << "Jvm opts str: " << jvm_opts << std::endl; if (*jvm_opts == '\0') return NULL; int num_of_opts = 1; - for (const char* p = jvm_opts; *p; p++) { + for (const char *p = jvm_opts; *p; p++) { if (*p == ' ') num_of_opts++; } @@ -54,13 +54,13 @@ JavaVM* CreateJavaVM(const std::string& jvm_options) { if (num_of_opts == 0) return NULL; - JavaVM* jvm = NULL; - JNIEnv* env = NULL; + JavaVM *jvm = NULL; + JNIEnv *env = NULL; int i = 0; int status = 1; JavaVMInitArgs vm_args; - JavaVMOption* options = new JavaVMOption[num_of_opts]; + JavaVMOption *options = new JavaVMOption[num_of_opts]; memset(options, 0, sizeof(JavaVMOption) * num_of_opts); for (p = q = jvm_opts;; p++) { @@ -68,11 +68,11 @@ JavaVM* CreateJavaVM(const std::string& jvm_options) { if (q >= p) { goto ret; } - char* opt = new char[p - q + 1]; + char *opt = new char[p - q + 1]; memcpy(opt, q, p - q); opt[p - q] = '\0'; options[i++].optionString = opt; - q = p + 1; // assume opts are separated by single space + q = p + 1; // assume opts are separated by single space if (*p == '\0') break; } @@ -83,14 +83,14 @@ JavaVM* CreateJavaVM(const std::string& jvm_options) { vm_args.nOptions = num_of_opts; vm_args.options = options; - status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); if (status == JNI_OK) { - LOG(INFO) << "Create java virtual machine successfully."; + std::cout << "Create java virtual machine successfully." << std::endl; } else if (status == JNI_EEXIST) { - VLOG(1) << "JNI evn already exists."; + std::cout << "JNI evn already exists." << std::endl; } else { - LOG(ERROR) << "Error, create java virtual machine failed. return JNI_CODE (" - << status << ")\n"; + std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")" << std::endl; } ret: @@ -102,40 +102,41 @@ JavaVM* CreateJavaVM(const std::string& jvm_options) { } // One process can only create jvm for once. -JavaVM* GetJavaVM(const std::string jvm_options = "") { +JavaVM *GetJavaVM(const std::string jvm_options = "") { if (_jvm == NULL) { // Try to find whether there exists one javaVM jsize nVMs; JNI_GetCreatedJavaVMs(NULL, 0, - &nVMs); // 1. just get the required array length - VLOG(1) << "Found " << nVMs << " VMs existing in this process."; - JavaVM** buffer = new JavaVM*[nVMs]; - JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + &nVMs); // 1. just get the required array length + std::cout << "Found " << nVMs << " VMs existing in this process." + << std::endl; + JavaVM **buffer = new JavaVM *[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data for (auto i = 0; i < nVMs; ++i) { if (buffer[i] != NULL) { _jvm = buffer[i]; - VLOG(1) << "Found index " << i << " VM non null " - << reinterpret_cast(_jvm); + std::cout << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm) << std::endl; return _jvm; } } _jvm = CreateJavaVM(jvm_options); - VLOG(1) << "Created JVM " << reinterpret_cast(_jvm); + std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; } return _jvm; } JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} -JNIEnvMark::JNIEnvMark(const std::string& jvm_options) : _env(NULL) { +JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) { if (!GetJavaVM(jvm_options)) { return; } int status = GetJavaVM(jvm_options) - ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); if (status != JNI_OK) { - LOG(ERROR) << "Error attach current thread: " << status; + std::cerr << "Error attach current thread: " << status << std::endl; } } @@ -145,45 +146,47 @@ JNIEnvMark::~JNIEnvMark() { } } -JNIEnv* JNIEnvMark::env() { return _env; } +JNIEnv *JNIEnvMark::env() { return _env; } -} // namespace jni +} // namespace jni #endif -std::vector list_files(const std::string& path) { +std::vector list_files(const std::string &path) { // list all files in the directory std::vector files; - for (const auto& entry : std::filesystem::directory_iterator(path)) { + for (const auto &entry : std::filesystem::directory_iterator(path)) { files.push_back(entry.path().string()); } return files; } -std::string GraphPlannerWrapper::expand_directory(const std::string& path) { +void iterate_over_director(const std::string& dir_or_path, std::vector& output_paths){ + if (dir_or_path.empty()) { + return; + } + if (std::filesystem::is_directory(dir_or_path)) { + auto files = list_files(dir_or_path); + output_paths.insert(output_paths.end(), files.begin(), files.end()); + } else { + output_paths.push_back(dir_or_path); + } +} + +std::string GraphPlannerWrapper::expand_directory(const std::string &path) { std::vector paths; std::string::size_type start = 0; std::string::size_type end = path.find(':'); while (end != std::string::npos) { auto sub_path = path.substr(start, end - start); - if (!sub_path.empty()) { - if (std::filesystem::is_directory(sub_path)) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } else { - paths.push_back(sub_path); - } - } + iterate_over_director(sub_path, paths); start = end + 1; end = path.find(':', start); } auto sub_path = path.substr(start); - if (!sub_path.empty()) { - auto files = list_files(sub_path); - paths.insert(paths.end(), files.begin(), files.end()); - } + iterate_over_director(sub_path, paths); std::stringstream ss; - for (const auto& p : paths) { + for (const auto &p : paths) { ss << p << ":"; } return ss.str(); @@ -192,11 +195,11 @@ std::string GraphPlannerWrapper::expand_directory(const std::string& path) { #if (GRAPH_PLANNER_JNI_INVOKER) std::string GraphPlannerWrapper::generate_jvm_options( - const std::string java_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json) { + const std::string java_path, const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) { auto expanded_java_path = expand_directory(java_path); - VLOG(10) << "Expanded java path: " << expanded_java_path; + std::cout << "Expanded java path: " << expanded_java_path << std::endl; std::string jvm_options = "-Djava.class.path=" + expanded_java_path; jvm_options += " -Djna.library.path=" + jna_path; jvm_options += " -Dgraph.schema=" + graph_schema_yaml; @@ -206,34 +209,28 @@ std::string GraphPlannerWrapper::generate_jvm_options( return jvm_options; } -struct Plan { - physical::PhysicalPlan physical_plan; - std::string result_schema; -} - Plan compilePlanJNI(jclass graph_planner_clz_, - jmethodID graph_planner_method_id_, - JNIEnv* env, - const std::string& compiler_config_path, - const std::string& cypher_query_string) { - jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + jmethodID graph_planner_method_id_, JNIEnv *env, + const std::string &compiler_config_path, + const std::string &cypher_query_string) { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), nullptr); Plan plan; if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { - LOG(ERROR) << "Invalid GraphPlannerWrapper."; + std::cerr << "Invalid GraphPlannerWrapper." << std::endl; return plan; } jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); // invoke jvm static function to get results as Object[] - jobjectArray resultArray = (jobjectArray) env->CallStaticObjectMethod( - graph_planner_clz_, graph_planner_method_id_, param1, param2); + jobjectArray resultArray = (jobjectArray)env->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2); if (env->ExceptionCheck()) { env->ExceptionDescribe(); env->ExceptionClear(); - LOG(ERROR) << "Error in calling GraphPlanner."; + std::cerr << "Error in calling GraphPlanner." << std::endl; return plan; } @@ -243,12 +240,12 @@ Plan compilePlanJNI(jclass graph_planner_clz_, jstring res2 = (jstring)env->GetObjectArrayElement(resultArray, 1); if (res == NULL || res2 == NULL) { - LOG(ERROR) << "Fail to generate plan."; + std::cerr << "Fail to generate plan." << std::endl; return plan; } - jbyte* str = env->GetByteArrayElements(res, NULL); + jbyte *str = env->GetByteArrayElements(res, NULL); jsize len = env->GetArrayLength(res); - LOG(INFO) << "Physical plan size: " << len; + std::cout << "Physical plan size: " << len; plan.physical_plan.ParseFromArray(str, len); plan.result_schema = env->GetStringUTFChars(res2, NULL); @@ -261,91 +258,91 @@ Plan compilePlanJNI(jclass graph_planner_clz_, env->DeleteLocalRef(res2); env->DeleteLocalRef(resultArray); - return physical_plan; + return plan; } #endif #if (!GRAPH_PLANNER_JNI_INVOKER) -void write_query_to_pipe(const std::string& path, - const std::string& query_str) { - LOG(INFO) << "write_query_to_pipe: " << path; +void write_query_to_pipe(const std::string &path, + const std::string &query_str) { + std::cout << "write_query_to_pipe: " << path << std::endl; // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); int fd_to_java = open(path.c_str(), O_WRONLY); if (fd_to_java < 0) { - LOG(ERROR) << "Fail to open pipe: " << path; + std::cerr << "Fail to open pipe: " << path << std::endl; return; } - LOG(INFO) << "open pipe done"; + std::cout << "open pipe done" << std::endl; auto len = write(fd_to_java, query_str.c_str(), query_str.size()); - if (len != (int) query_str.size()) { - LOG(ERROR) << "Fail to write query to pipe:" << len; + if (len != (int)query_str.size()) { + std::cerr << "Fail to write query to pipe:" << len << std::endl; return; } - LOG(INFO) << "write_query_to_pipe done: " << len; + std::cout << "write_query_to_pipe done: " << len << std::endl; close(fd_to_java); } -void write_query_to_file(const std::string& path, - const std::string& query_str) { +void write_query_to_file(const std::string &path, + const std::string &query_str) { std::ofstream query_file(path); query_file << query_str; query_file.close(); } -physical::PhysicalPlan readPhysicalPlan(const std::string& plan_str) { - VLOG(10) << "plan str size: " << plan_str.size(); +physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) { + std::cout << "plan str size: " << plan_str.size() << std::endl; physical::PhysicalPlan plan; if (!plan.ParseFromString(plan_str)) { - LOG(ERROR) << "Fail to parse physical plan."; + std::cerr << "Fail to parse physical plan." << std::endl; return physical::PhysicalPlan(); } return plan; } -physical::PhysicalPlan compilePlanSubprocess( - const std::string& class_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json, - const std::string& compiler_config_path, - const std::string& cypher_query_string) { +physical::PhysicalPlan +compilePlanSubprocess(const std::string &class_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json, + const std::string &compiler_config_path, + const std::string &cypher_query_string) { physical::PhysicalPlan physical_plan; auto random_prefix = std::to_string( std::chrono::system_clock::now().time_since_epoch().count()); std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; - VLOG(10) << "dst_query_path: " << dst_query_path - << " dst_output_file: " << dst_output_file; + std::cout << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file << std::endl; mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); pid_t pid = fork(); if (pid == 0) { - const char* const command_string_array[] = { - "java", - "-cp", - class_path.c_str(), - jna_path.c_str(), - graph_schema_yaml.c_str(), - graph_statistic_json.c_str(), - "com.alibaba.graphscope.common.ir.tools.GraphPlanner", - compiler_config_path.c_str(), - dst_query_path.c_str(), - dst_output_file.c_str(), - "/tmp/temp.cypher.yaml", - NULL}; + const char *const command_string_array[] = {"java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + GRAPH_PLANNER_FULL_NAME, + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; execvp(command_string_array[0], - const_cast(command_string_array)); + const_cast(command_string_array)); } else if (pid < 0) { - LOG(ERROR) << "Error in fork."; + std::cerr << "Error in fork." << std::endl; } else { write_query_to_pipe(dst_query_path, cypher_query_string); int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); if (fd_from_java < 0) { - LOG(ERROR) << "Fail to open pipe: " << dst_output_file; + std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; return physical_plan; } std::vector stored_buffer; @@ -364,7 +361,7 @@ physical::PhysicalPlan compilePlanSubprocess( int status; waitpid(pid, &status, 0); if (status != 0) { - LOG(ERROR) << "Error in running command."; + std::cerr << "Error in running command." << std::endl; } } unlink(dst_query_path.c_str()); @@ -373,9 +370,8 @@ physical::PhysicalPlan compilePlanSubprocess( } #endif -physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( - const std::string& compiler_config_path, - const std::string& cypher_query_string) { +Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string) { #if (GRAPH_PLANNER_JNI_INVOKER) return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, jni_wrapper_.env(), compiler_config_path, @@ -387,4 +383,4 @@ physical::PhysicalPlan GraphPlannerWrapper::CompilePlan( #endif } -} // namespace gs +} // namespace gs diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.h b/interactive_engine/compiler/src/main/cpp/graph_planner.h index d95a0792a50f..9fcb720c59a3 100644 --- a/interactive_engine/compiler/src/main/cpp/graph_planner.h +++ b/interactive_engine/compiler/src/main/cpp/graph_planner.h @@ -16,60 +16,65 @@ limitations under the License. #ifndef PLANNER_GRAPH_PLANNER_H_ #define PLANNER_GRAPH_PLANNER_H_ -#include #include #include +#include #include #include #include "flex/proto_generated_gie/physical.pb.h" -#include "glog/logging.h" - #ifndef GRAPH_PLANNER_JNI_INVOKER -#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess #endif namespace gs { +struct Plan { + physical::PhysicalPlan physical_plan; + std::string result_schema; +}; + #if (GRAPH_PLANNER_JNI_INVOKER) namespace jni { struct JNIEnvMark { - JNIEnv* _env; + JNIEnv *_env; JNIEnvMark(); - JNIEnvMark(const std::string& jvm_options); + JNIEnvMark(const std::string &jvm_options); ~JNIEnvMark(); - JNIEnv* env(); + JNIEnv *env(); }; -} // namespace jni +} // namespace jni #endif class GraphPlannerWrapper { - public: - static constexpr const char* kGraphPlannerClass = +public: + static constexpr const char *kGraphPlannerClass = "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; - static constexpr const char* kGraphPlannerMethod = "generatePhysicalPlan"; - static constexpr const char* kGraphPlannerMethodSignature = - "(Ljava/lang/String;Ljava/lang/String;)[B"; - - GraphPlannerWrapper(const std::string java_path, const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json = "") + static constexpr const char *GRAPH_PLANNER_FULL_NAME = + "com.alibaba.graphscope.common.ir.tools.GraphPlanner"; + static constexpr const char *kGraphPlannerMethod = "generatePhysicalPlan"; + static constexpr const char *kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; + + GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json = "") #if (GRAPH_PLANNER_JNI_INVOKER) : jni_wrapper_(generate_jvm_options( java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); if (clz == NULL) { - LOG(ERROR) << "Fail to find class: " << kGraphPlannerClass; + std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; return; } - graph_planner_clz_ = (jclass) jni_wrapper_.env()->NewGlobalRef(clz); + graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); if (j_method_id == NULL) { - LOG(ERROR) << "Fail to find method: " << kGraphPlannerMethod; + std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; return; } graph_planner_method_id_ = j_method_id; @@ -94,8 +99,8 @@ class GraphPlannerWrapper { #if (GRAPH_PLANNER_JNI_INVOKER) return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; #else - return true; // just return true, since we don't have a way to check the - // validity when calling via subprocess. + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. #endif } @@ -106,19 +111,19 @@ class GraphPlannerWrapper { * @param cypher_query_string The cypher query string. * @return physical plan in string. */ - physical::PhysicalPlan CompilePlan(const std::string& compiler_config_path, - const std::string& cypher_query_string); + Plan CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string); - private: +private: std::string generate_jvm_options(const std::string java_path, - const std::string& jna_path, - const std::string& graph_schema_yaml, - const std::string& graph_statistic_json); + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); // physical::PhysicalPlan compilePlanJNI(const std::string& // compiler_config_path, // const std::string& // cypher_query_string); - std::string expand_directory(const std::string& path); + std::string expand_directory(const std::string &path); #if (GRAPH_PLANNER_JNI_INVOKER) // We need to list all files in the directory, if exists. // The reason why we need to list all files in the directory is that @@ -136,6 +141,6 @@ class GraphPlannerWrapper { std::string graph_statistic_json_; #endif }; -} // namespace gs +} // namespace gs -#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file +#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/cpp/test/test.cc b/interactive_engine/compiler/src/main/cpp/test/test.cc new file mode 100644 index 000000000000..2d28e884dfb9 --- /dev/null +++ b/interactive_engine/compiler/src/main/cpp/test/test.cc @@ -0,0 +1,83 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include "graph_planner.h" + +#include + +std::string get_dir_name() { + // Get the directory of this source file + std::string current_dir = __FILE__; + size_t pos = current_dir.find_last_of("/"); + current_dir = current_dir.substr(0, pos); + return current_dir; +} + +void check_path_exits(const std::string& path) { + // split path by ':' + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) { + auto sub_path = path.substr(start, end - start); + paths.push_back(sub_path); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + paths.push_back(sub_path); + + for (const auto& p : paths) { + struct stat buffer; + if (stat(p.c_str(), &buffer) != 0) { + std::cerr << "Path not exists: " << p << std::endl; + exit(1); + } + } + std::cout<< "Path exists: " << path << std::endl; +} + +int main(int argc, char **argv) { + auto current_dir = get_dir_name(); + std::string java_class_path = + current_dir + "/../../../../target/compiler-0.0.1-SNAPSHOT.jar"; + java_class_path+= ":" + current_dir + "/../../../../target/libs/"; + std::string jna_class_path = + current_dir + "/../../../../../executor/ir/target/release/"; + std::string graph_schema_yaml = + current_dir + + "/../../../../../../flex/interactive/examples/modern_graph/graph.yaml"; + std::string graph_statistic_json = + current_dir + + "/../../../test/resources/statistics/modern_statistics.json"; + + // check director or file exists + check_path_exits(java_class_path); + check_path_exits(jna_class_path); + check_path_exits(graph_schema_yaml); + check_path_exits(graph_statistic_json); + + gs::GraphPlannerWrapper graph_planner_wrapper( + java_class_path, jna_class_path, graph_schema_yaml, graph_statistic_json); + + std::string cypher_query_string = "MATCH (a:person) RETURN a.name"; + std::string config_path = + current_dir + + "/../../../../../../flex/tests/hqps/interactive_config_test.yaml"; + auto plan = + graph_planner_wrapper.CompilePlan(config_path, cypher_query_string); + std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; + std::cout << "schema: " << plan.result_schema << std::endl; + return 0; +} \ No newline at end of file diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index ebe5c8063194..cddb06fe0274 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -49,36 +49,6 @@ compiler - - graph-planner-jni - - false - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - make - - run - - compile - - - - - - - - - - - - - graphscope @@ -728,12 +698,6 @@ ${interactive.sdk.version} ${interactive.sdk.classifier} - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - maven-plugin - @@ -921,6 +885,11 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.0.0 + From 837a07eaefd0a45e900fe4074ca21cdd8aebd2c2 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 11 Dec 2024 13:56:51 +0800 Subject: [PATCH 08/27] set cxx_standard 17 to use filesystem Committed-by: xiaolei.zl from Dev container --- interactive_engine/compiler/src/main/cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt index 53bc20f0f9fa..23949fc6038a 100644 --- a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt +++ b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt @@ -19,6 +19,7 @@ project(COMPILER-JNI option(BUILD_TEST "Whether to build test" ON) set(JAVA_AWT_INCLUDE_PATH NotNeeded) +set(CMAKE_CXX_STANDARD 17) find_package(JNI QUIET) if (JNI_FOUND) include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) From ad4940231d978b7a0a65e73978d7f8ad088b9fea Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 14:20:46 +0800 Subject: [PATCH 09/27] add graph-planner-jni in root pom --- interactive_engine/compiler/build_native.xml | 2 +- interactive_engine/pom.xml | 30 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml index 049909547f46..ac17a454378c 100644 --- a/interactive_engine/compiler/build_native.xml +++ b/interactive_engine/compiler/build_native.xml @@ -41,7 +41,7 @@ - + diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index cddb06fe0274..f79b9c0666ad 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -49,6 +49,36 @@ compiler + + graph-planner-jni + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + + run + + compile + + + + + + + + + + + + + graphscope From f55eed667df913cafc9d7552a124da3d5ed05a3c Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 14:34:24 +0800 Subject: [PATCH 10/27] remove the cargo build phase from mvn build --- interactive_engine/compiler/build_ir_core.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/interactive_engine/compiler/build_ir_core.sh b/interactive_engine/compiler/build_ir_core.sh index 532e2f0a5f64..3a27f9725c6d 100755 --- a/interactive_engine/compiler/build_ir_core.sh +++ b/interactive_engine/compiler/build_ir_core.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -e -set -x - -if $(! command -v cargo &> /dev/null) -then - echo "cargo not exit, skip compile" -else - cd ../executor/ir/core - cargo build --release -fi +#set -e +#set -x +# +#if $(! command -v cargo &> /dev/null) +#then +# echo "cargo not exit, skip compile" +#else +# cd ../executor/ir/core +# cargo build --release +#fi From ce0bbc4e0ef750a8ccf83566977294a8712644a5 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 14:54:49 +0800 Subject: [PATCH 11/27] fix compiling errors in mac os --- interactive_engine/compiler/build_native.xml | 8 ++++---- interactive_engine/compiler/src/main/cpp/CMakeLists.txt | 4 +++- interactive_engine/compiler/src/main/cpp/graph_planner.h | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml index ac17a454378c..c6ebb57c08fc 100644 --- a/interactive_engine/compiler/build_native.xml +++ b/interactive_engine/compiler/build_native.xml @@ -28,9 +28,9 @@ - - - + + + @@ -45,7 +45,7 @@ - + diff --git a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt index 23949fc6038a..e4157c2aa8ae 100644 --- a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt +++ b/interactive_engine/compiler/src/main/cpp/CMakeLists.txt @@ -59,6 +59,8 @@ protobuf_generate(APPEND_PATH add_library(ir_proto SHARED ${PROTO_SRCS_GIE}) +target_link_libraries(ir_proto ${Protobuf_LIBRARIES}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) file(GLOB_RECURSE PLANNER_SRC_FILES "graph_planner.cc") @@ -69,4 +71,4 @@ target_include_directories(graph_planner PUBLIC $ #include +#include #include #include #include @@ -143,4 +144,4 @@ class GraphPlannerWrapper { }; } // namespace gs -#endif // PLANNER_GRAPH_PLANNER_H_ \ No newline at end of file +#endif // PLANNER_GRAPH_PLANNER_H_ From 8e3cb54633246dd68a7672667e7da658a285131e Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 15:03:24 +0800 Subject: [PATCH 12/27] minor fix --- interactive_engine/compiler/build_native.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml index c6ebb57c08fc..f8ff1e84fe02 100644 --- a/interactive_engine/compiler/build_native.xml +++ b/interactive_engine/compiler/build_native.xml @@ -48,8 +48,8 @@ - - - + + + From 7ffc2e4a61d359c12b19340050992cd1cbf56afb Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 20:01:49 +0800 Subject: [PATCH 13/27] fix bugs of jni building and add readme --- .../assembly/graph-planner-jni.xml | 72 + interactive_engine/assembly/pom.xml | 17 + interactive_engine/compiler/build_native.xml | 12 +- .../compiler/conf/ir.compiler.properties | 6 +- interactive_engine/compiler/diff | 1199 +++++++++++++++++ interactive_engine/compiler/pom.xml | 31 +- .../compiler/src/main/cpp/README.md | 3 - .../compiler/src/main/cpp/graph_planner.cc | 386 ------ .../compiler/src/main/cpp/graph_planner.h | 147 -- .../compiler/src/main/cpp/test/test.cc | 83 -- .../com/alibaba/graphscope/GraphServer.java | 4 +- .../graphscope/common/config/YamlConfigs.java | 2 +- .../meta/procedure/StoredProcedureMeta.java | 2 +- .../common/ir/tools/GraphPlanner.java | 17 - .../common/jna/_native/GraphPlannerJNI.java | 54 + .../common/jna/_native/JNIPlan.java | 37 + .../src/main/{cpp => native}/CMakeLists.txt | 0 .../compiler/src/main/native/README.md | 85 ++ .../compiler/src/main/native/graph_planner.cc | 456 +++++++ .../compiler/src/main/native/graph_planner.h | 167 +++ .../compiler/src/main/native/test/test.cc | 87 ++ .../alibaba/graphscope/common/ir/JNITest.java | 11 +- .../config/interactive_config_test.yaml | 46 + interactive_engine/pom.xml | 24 +- 24 files changed, 2245 insertions(+), 703 deletions(-) create mode 100644 interactive_engine/assembly/graph-planner-jni.xml create mode 100644 interactive_engine/compiler/diff delete mode 100644 interactive_engine/compiler/src/main/cpp/README.md delete mode 100644 interactive_engine/compiler/src/main/cpp/graph_planner.cc delete mode 100644 interactive_engine/compiler/src/main/cpp/graph_planner.h delete mode 100644 interactive_engine/compiler/src/main/cpp/test/test.cc create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java rename interactive_engine/compiler/src/main/{cpp => native}/CMakeLists.txt (100%) create mode 100644 interactive_engine/compiler/src/main/native/README.md create mode 100644 interactive_engine/compiler/src/main/native/graph_planner.cc create mode 100644 interactive_engine/compiler/src/main/native/graph_planner.h create mode 100644 interactive_engine/compiler/src/main/native/test/test.cc create mode 100644 interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml new file mode 100644 index 000000000000..3413eed2a566 --- /dev/null +++ b/interactive_engine/assembly/graph-planner-jni.xml @@ -0,0 +1,72 @@ + + graph-planner-jni + + tar.gz + + + + + ${project.parent.basedir}/executor/ir/target/release + + libir_core.* + + native + + + ${project.parent.basedir}/target/native/ + + libgraph_planner.* + + native + + + ${project.parent.basedir}/target/native/ + + test_graph_planner + + bin + 0755 + + + ${project.parent.basedir}/compiler/target/libs/ + libs + + + ${project.parent.basedir}/compiler/target/ + + compiler-0.0.1-SNAPSHOT.jar + + libs + + + ${project.parent.basedir}/compiler/conf + conf + + * + + + + ${project.parent.basedir}/../flex/interactive/examples/modern_graph/ + conf + + graph.yaml + + + + ${project.parent.basedir}/compiler/src/test/resources/statistics/ + conf + + modern_statistics.json + + + + ${project.parent.basedir}/compiler/src/test/resources/config/ + conf + + interactive_config_test.yaml + + + + diff --git a/interactive_engine/assembly/pom.xml b/interactive_engine/assembly/pom.xml index 39b0aa02ea32..a08a9a013d24 100644 --- a/interactive_engine/assembly/pom.xml +++ b/interactive_engine/assembly/pom.xml @@ -71,5 +71,22 @@ + + graph-planner-jni + + + + org.apache.maven.plugins + maven-assembly-plugin + + graph-planner-jni + + graph-planner-jni.xml + + + + + + diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml index f8ff1e84fe02..0ffbb765fb46 100644 --- a/interactive_engine/compiler/build_native.xml +++ b/interactive_engine/compiler/build_native.xml @@ -27,10 +27,6 @@ - - - - @@ -41,15 +37,15 @@ - + - - - + + + diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 992f828a2b2e..5a13dc9d4f43 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json graph.store: exp graph.planner.is.on: true -graph.planner.opt: RBO +graph.planner.opt: CBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule # set statistics access uri -# graph.statistics: src/test/resources/statistics/modern_statistics.json +graph.statistics: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: @@ -60,7 +60,7 @@ calcite.default.charset: UTF-8 # gremlin.script.language.name: antlr_gremlin_traversal # the output plan format, can be ffi(default) or proto -# graph.physical.opt: ffi +graph.physical.opt: proto # set the max capacity of the result streaming buffer for each query # per.query.stream.buffer.max.capacity: 256 diff --git a/interactive_engine/compiler/diff b/interactive_engine/compiler/diff new file mode 100644 index 000000000000..30e8edec8263 --- /dev/null +++ b/interactive_engine/compiler/diff @@ -0,0 +1,1199 @@ +diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml +index 0e4495ca64..e8efca65d7 100644 +--- a/interactive_engine/assembly/graph-planner-jni.xml ++++ b/interactive_engine/assembly/graph-planner-jni.xml +@@ -16,6 +16,16 @@ + + + ${project.parent.basedir}/target/native/ ++ ++ libgraph_planner.* ++ ++ native ++ ++ ++ ${project.parent.basedir}/target/native/ ++ ++ test_graph_planner ++ + bin + 0755 + +@@ -23,6 +33,13 @@ + ${project.parent.basedir}/compiler/target/libs/ + libs + ++ ++ ${project.parent.basedir}/compiler/target/ ++ ++ compiler-0.0.1-SNAPSHOT.jar ++ ++ libs ++ + + ${project.parent.basedir}/compiler/conf + conf +diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties +index 992f828a2b..5a13dc9d4f 100644 +--- a/interactive_engine/compiler/conf/ir.compiler.properties ++++ b/interactive_engine/compiler/conf/ir.compiler.properties +@@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json + graph.store: exp + + graph.planner.is.on: true +-graph.planner.opt: RBO ++graph.planner.opt: CBO + graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule + + # set statistics access uri +-# graph.statistics: src/test/resources/statistics/modern_statistics.json ++graph.statistics: src/test/resources/statistics/modern_statistics.json + + # set stored procedures directory path + # graph.stored.procedures: +@@ -60,7 +60,7 @@ calcite.default.charset: UTF-8 + # gremlin.script.language.name: antlr_gremlin_traversal + + # the output plan format, can be ffi(default) or proto +-# graph.physical.opt: ffi ++graph.physical.opt: proto + + # set the max capacity of the result streaming buffer for each query + # per.query.stream.buffer.max.capacity: 256 +diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +index 991ae7b89d..77dc36d2a8 100644 +--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java ++++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +@@ -249,23 +249,6 @@ public class GraphPlanner { + return summary; + } + +- public static Object[] generatePhysicalPlan(String config_path, String query_string) +- throws Exception { +- Summary summary = generatePlan(config_path, query_string); +- LogicalPlan logicalPlan = summary.getLogicalPlan(); +- PhysicalPlan physicalPlan = summary.physicalPlan; +- Configs extraConfigs = createExtraConfigs(null); +- StoredProcedureMeta procedureMeta = +- new StoredProcedureMeta( +- extraConfigs, +- query_string, +- logicalPlan.getOutputType(), +- logicalPlan.getDynamicParams()); +- ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); +- StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); +- return new Object[] {physicalPlan.getContent(), new String(metaStream.toByteArray())}; +- } +- + public static void main(String[] args) throws Exception { + if (args.length < 4 + || args[0].isEmpty() +diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java +index bea8926ed7..ee1766e855 100644 +--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java ++++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java +@@ -1,2 +1,55 @@ +-package com.alibaba.graphscope.common.jna._native;public class GraphPlannerJNI { ++/* ++ * ++ * * Copyright 2020 Alibaba Group Holding Limited. ++ * * ++ * * Licensed under the Apache License, Version 2.0 (the "License"); ++ * * you may not use this file except in compliance with the License. ++ * * You may obtain a copy of the License at ++ * * ++ * * http://www.apache.org/licenses/LICENSE-2.0 ++ * * ++ * * Unless required by applicable law or agreed to in writing, software ++ * * distributed under the License is distributed on an "AS IS" BASIS, ++ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * * See the License for the specific language governing permissions and ++ * * limitations under the License. ++ * ++ */ ++ ++package com.alibaba.graphscope.common.jna._native; ++ ++import com.alibaba.graphscope.common.config.Configs; ++import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; ++import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; ++import com.alibaba.graphscope.common.ir.tools.GraphPlanner; ++import com.alibaba.graphscope.common.ir.tools.LogicalPlan; ++import com.google.common.collect.ImmutableMap; ++ ++import java.io.ByteArrayOutputStream; ++ ++public class GraphPlannerJNI { ++ /** ++ * Provide a java-side implementation to compile the query in string to a physical plan ++ * @param config ++ * @param query ++ * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, ++ * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, ++ * resultSchemaYaml defines the result specification of the query in yaml format ++ * @throws Exception ++ */ ++ public static JNIPlan compilePlan(String config, String query) ++ throws Exception { ++ GraphPlanner.Summary summary = GraphPlanner.generatePlan(config, query); ++ LogicalPlan logicalPlan = summary.getLogicalPlan(); ++ PhysicalPlan physicalPlan = summary.getPhysicalPlan(); ++ StoredProcedureMeta procedureMeta = ++ new StoredProcedureMeta( ++ new Configs(ImmutableMap.of()), ++ query, ++ logicalPlan.getOutputType(), ++ logicalPlan.getDynamicParams()); ++ ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); ++ StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); ++ return new JNIPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); ++ } + } +diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java +index 93d4e3615a..689c9c74ef 100644 +--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java ++++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java +@@ -1,2 +1,37 @@ +-package com.alibaba.graphscope.common.jna._native;public class JNIPlan { ++/* ++ * ++ * * Copyright 2020 Alibaba Group Holding Limited. ++ * * ++ * * Licensed under the Apache License, Version 2.0 (the "License"); ++ * * you may not use this file except in compliance with the License. ++ * * You may obtain a copy of the License at ++ * * ++ * * http://www.apache.org/licenses/LICENSE-2.0 ++ * * ++ * * Unless required by applicable law or agreed to in writing, software ++ * * distributed under the License is distributed on an "AS IS" BASIS, ++ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * * See the License for the specific language governing permissions and ++ * * limitations under the License. ++ * ++ */ ++ ++package com.alibaba.graphscope.common.jna._native; ++ ++public class JNIPlan { ++ public final byte[] physicalBytes; ++ public final String resultSchemaYaml; ++ ++ public JNIPlan(byte[] physicalBytes, String resultSchemaYaml) { ++ this.physicalBytes = physicalBytes; ++ this.resultSchemaYaml = resultSchemaYaml; ++ } ++ ++ public byte[] getPhysicalBytes() { ++ return physicalBytes; ++ } ++ ++ public String getResultSchemaYaml() { ++ return resultSchemaYaml; ++ } + } +diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc +index 417169a130..0b4a304fd6 100644 +--- a/interactive_engine/compiler/src/main/native/graph_planner.cc ++++ b/interactive_engine/compiler/src/main/native/graph_planner.cc +@@ -22,365 +22,429 @@ limitations under the License. + #include "graph_planner.h" + #include + +-namespace gs { ++namespace gs ++{ + #if (GRAPH_PLANNER_JNI_INVOKER) +-namespace jni { +- +-static JavaVM *_jvm = NULL; +- +-JavaVM *CreateJavaVM(const std::string &jvm_options) { +- const char *p, *q; +- const char *jvm_opts; +- if (jvm_options.empty()) { +- jvm_opts = getenv("FLEX_JVM_OPTS"); +- } else { +- jvm_opts = jvm_options.c_str(); +- } +- if (jvm_opts == NULL) { +- std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; +- return NULL; +- } +- std::cout << "Jvm opts str: " << jvm_opts << std::endl; ++ namespace jni ++ { ++ ++ static JavaVM *_jvm = NULL; ++ ++ JavaVM *CreateJavaVM(const std::string &jvm_options) ++ { ++ const char *p, *q; ++ const char *jvm_opts; ++ if (jvm_options.empty()) ++ { ++ jvm_opts = getenv("FLEX_JVM_OPTS"); ++ } ++ else ++ { ++ jvm_opts = jvm_options.c_str(); ++ } ++ if (jvm_opts == NULL) ++ { ++ std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; ++ return NULL; ++ } ++ std::cout << "Jvm opts str: " << jvm_opts << std::endl; + +- if (*jvm_opts == '\0') +- return NULL; ++ if (*jvm_opts == '\0') ++ return NULL; + +- int num_of_opts = 1; +- for (const char *p = jvm_opts; *p; p++) { +- if (*p == ' ') +- num_of_opts++; +- } ++ int num_of_opts = 1; ++ for (const char *p = jvm_opts; *p; p++) ++ { ++ if (*p == ' ') ++ num_of_opts++; ++ } + +- if (num_of_opts == 0) +- return NULL; ++ if (num_of_opts == 0) ++ return NULL; ++ ++ JavaVM *jvm = NULL; ++ JNIEnv *env = NULL; ++ int i = 0; ++ int status = 1; ++ JavaVMInitArgs vm_args; ++ ++ JavaVMOption *options = new JavaVMOption[num_of_opts]; ++ memset(options, 0, sizeof(JavaVMOption) * num_of_opts); ++ ++ for (p = q = jvm_opts;; p++) ++ { ++ if (*p == ' ' || *p == '\0') ++ { ++ if (q >= p) ++ { ++ goto ret; ++ } ++ char *opt = new char[p - q + 1]; ++ memcpy(opt, q, p - q); ++ opt[p - q] = '\0'; ++ options[i++].optionString = opt; ++ q = p + 1; // assume opts are separated by single space ++ if (*p == '\0') ++ break; ++ } ++ } + +- JavaVM *jvm = NULL; +- JNIEnv *env = NULL; +- int i = 0; +- int status = 1; +- JavaVMInitArgs vm_args; ++ memset(&vm_args, 0, sizeof(vm_args)); ++ vm_args.version = JNI_VERSION_1_8; ++ vm_args.nOptions = num_of_opts; ++ vm_args.options = options; + +- JavaVMOption *options = new JavaVMOption[num_of_opts]; +- memset(options, 0, sizeof(JavaVMOption) * num_of_opts); ++ status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); ++ if (status == JNI_OK) ++ { ++ std::cout << "Create java virtual machine successfully." << std::endl; ++ } ++ else if (status == JNI_EEXIST) ++ { ++ std::cout << "JNI evn already exists." << std::endl; ++ } ++ else ++ { ++ std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" ++ << status << ")" << std::endl; ++ } + +- for (p = q = jvm_opts;; p++) { +- if (*p == ' ' || *p == '\0') { +- if (q >= p) { +- goto ret; ++ ret: ++ for (int i = 0; i < num_of_opts; i++) ++ { ++ delete[] options[i].optionString; + } +- char *opt = new char[p - q + 1]; +- memcpy(opt, q, p - q); +- opt[p - q] = '\0'; +- options[i++].optionString = opt; +- q = p + 1; // assume opts are separated by single space +- if (*p == '\0') +- break; ++ delete[] options; ++ return jvm; + } +- } +- +- memset(&vm_args, 0, sizeof(vm_args)); +- vm_args.version = JNI_VERSION_1_8; +- vm_args.nOptions = num_of_opts; +- vm_args.options = options; +- +- status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); +- if (status == JNI_OK) { +- std::cout << "Create java virtual machine successfully." << std::endl; +- } else if (status == JNI_EEXIST) { +- std::cout << "JNI evn already exists." << std::endl; +- } else { +- std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" +- << status << ")" << std::endl; +- } + +-ret: +- for (int i = 0; i < num_of_opts; i++) { +- delete[] options[i].optionString; +- } +- delete[] options; +- return jvm; +-} +- +-// One process can only create jvm for once. +-JavaVM *GetJavaVM(const std::string jvm_options = "") { +- if (_jvm == NULL) { +- // Try to find whether there exists one javaVM +- jsize nVMs; +- JNI_GetCreatedJavaVMs(NULL, 0, +- &nVMs); // 1. just get the required array length +- std::cout << "Found " << nVMs << " VMs existing in this process." +- << std::endl; +- JavaVM **buffer = new JavaVM *[nVMs]; +- JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data +- for (auto i = 0; i < nVMs; ++i) { +- if (buffer[i] != NULL) { +- _jvm = buffer[i]; +- std::cout << "Found index " << i << " VM non null " +- << reinterpret_cast(_jvm) << std::endl; +- return _jvm; ++ // One process can only create jvm for once. ++ JavaVM *GetJavaVM(const std::string jvm_options = "") ++ { ++ if (_jvm == NULL) ++ { ++ // Try to find whether there exists one javaVM ++ jsize nVMs; ++ JNI_GetCreatedJavaVMs(NULL, 0, ++ &nVMs); // 1. just get the required array length ++ std::cout << "Found " << nVMs << " VMs existing in this process." ++ << std::endl; ++ JavaVM **buffer = new JavaVM *[nVMs]; ++ JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data ++ for (auto i = 0; i < nVMs; ++i) ++ { ++ if (buffer[i] != NULL) ++ { ++ _jvm = buffer[i]; ++ std::cout << "Found index " << i << " VM non null " ++ << reinterpret_cast(_jvm) << std::endl; ++ return _jvm; ++ } ++ } ++ _jvm = CreateJavaVM(jvm_options); ++ std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; + } ++ return _jvm; + } +- _jvm = CreateJavaVM(jvm_options); +- std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; +- } +- return _jvm; +-} + +-JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} ++ JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + +-JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) { +- if (!GetJavaVM(jvm_options)) { +- return; +- } +- int status = +- GetJavaVM(jvm_options) +- ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); +- if (status != JNI_OK) { +- std::cerr << "Error attach current thread: " << status << std::endl; +- } +-} ++ JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) ++ { ++ if (!GetJavaVM(jvm_options)) ++ { ++ return; ++ } ++ int status = ++ GetJavaVM(jvm_options) ++ ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); ++ if (status != JNI_OK) ++ { ++ std::cerr << "Error attach current thread: " << status << std::endl; ++ } ++ } + +-JNIEnvMark::~JNIEnvMark() { +- if (_env) { +- GetJavaVM()->DetachCurrentThread(); +- } +-} ++ JNIEnvMark::~JNIEnvMark() ++ { ++ if (_env) ++ { ++ GetJavaVM()->DetachCurrentThread(); ++ } ++ } + +-JNIEnv *JNIEnvMark::env() { return _env; } ++ JNIEnv *JNIEnvMark::env() { return _env; } + +-} // namespace jni ++ } // namespace jni + + #endif + +-std::vector list_files(const std::string &path) { +- // list all files in the directory +- std::vector files; +- for (const auto &entry : std::filesystem::directory_iterator(path)) { +- files.push_back(entry.path().string()); ++ std::vector list_files(const std::string &path) ++ { ++ // list all files in the directory ++ std::vector files; ++ for (const auto &entry : std::filesystem::directory_iterator(path)) ++ { ++ files.push_back(entry.path().string()); ++ } ++ return files; + } +- return files; +-} + +-void iterate_over_director(const std::string& dir_or_path, std::vector& output_paths){ +- if (dir_or_path.empty()) { +- return; ++ void iterate_over_director(const std::string &dir_or_path, std::vector &output_paths) ++ { ++ if (dir_or_path.empty()) ++ { ++ return; ++ } ++ if (std::filesystem::is_directory(dir_or_path)) ++ { ++ auto files = list_files(dir_or_path); ++ output_paths.insert(output_paths.end(), files.begin(), files.end()); ++ } ++ else ++ { ++ output_paths.push_back(dir_or_path); ++ } + } +- if (std::filesystem::is_directory(dir_or_path)) { +- auto files = list_files(dir_or_path); +- output_paths.insert(output_paths.end(), files.begin(), files.end()); +- } else { +- output_paths.push_back(dir_or_path); +- } +-} +- +-std::string GraphPlannerWrapper::expand_directory(const std::string &path) { +- std::vector paths; +- std::string::size_type start = 0; +- std::string::size_type end = path.find(':'); +- while (end != std::string::npos) { +- auto sub_path = path.substr(start, end - start); ++ ++ std::string GraphPlannerWrapper::expand_directory(const std::string &path) ++ { ++ std::vector paths; ++ std::string::size_type start = 0; ++ std::string::size_type end = path.find(':'); ++ while (end != std::string::npos) ++ { ++ auto sub_path = path.substr(start, end - start); ++ iterate_over_director(sub_path, paths); ++ start = end + 1; ++ end = path.find(':', start); ++ } ++ auto sub_path = path.substr(start); + iterate_over_director(sub_path, paths); +- start = end + 1; +- end = path.find(':', start); +- } +- auto sub_path = path.substr(start); +- iterate_over_director(sub_path, paths); +- std::stringstream ss; +- for (const auto &p : paths) { +- ss << p << ":"; ++ std::stringstream ss; ++ for (const auto &p : paths) ++ { ++ ss << p << ":"; ++ } ++ return ss.str(); + } +- return ss.str(); +-} + + #if (GRAPH_PLANNER_JNI_INVOKER) + +-std::string GraphPlannerWrapper::generate_jvm_options( +- const std::string java_path, const std::string &jna_path, +- const std::string &graph_schema_yaml, +- const std::string &graph_statistic_json) { +- auto expanded_java_path = expand_directory(java_path); +- std::cout << "Expanded java path: " << expanded_java_path << std::endl; +- std::string jvm_options = "-Djava.class.path=" + expanded_java_path; +- jvm_options += " -Djna.library.path=" + jna_path; +- jvm_options += " -Dgraph.schema=" + graph_schema_yaml; +- if (!graph_statistic_json.empty()) { +- jvm_options += " -Dgraph.statistic=" + graph_statistic_json; +- } +- return jvm_options; +-} +- +-Plan compilePlanJNI(jclass graph_planner_clz_, +- jmethodID graph_planner_method_id_, JNIEnv *env, +- const std::string &compiler_config_path, +- const std::string &cypher_query_string) { +- jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), +- nullptr); +- Plan plan; +- if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { +- std::cerr << "Invalid GraphPlannerWrapper." << std::endl; +- return plan; ++ std::string GraphPlannerWrapper::generate_jvm_options( ++ const std::string java_path, const std::string &jna_path, ++ const std::string &graph_schema_yaml, ++ const std::string &graph_statistic_json) ++ { ++ auto expanded_java_path = expand_directory(java_path); ++ std::cout << "Expanded java path: " << expanded_java_path << std::endl; ++ std::string jvm_options = "-Djava.class.path=" + expanded_java_path; ++ jvm_options += " -Djna.library.path=" + jna_path; ++ jvm_options += " -Dgraph.schema=" + graph_schema_yaml; ++ if (!graph_statistic_json.empty()) ++ { ++ jvm_options += " -Dgraph.statistic=" + graph_statistic_json; ++ } ++ return jvm_options; + } +- jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); +- jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + +- // invoke jvm static function to get results as Object[] +- jobjectArray resultArray = (jobjectArray)env->CallStaticObjectMethod( +- graph_planner_clz_, graph_planner_method_id_, param1, param2); ++ Plan compilePlanJNI(jclass graph_planner_clz_, ++ jmethodID graph_planner_method_id_, JNIEnv *env, ++ const std::string &compiler_config_path, ++ const std::string &cypher_query_string) ++ { ++ jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), ++ nullptr); ++ Plan plan; ++ if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) ++ { ++ std::cerr << "Invalid GraphPlannerWrapper." << std::endl; ++ return plan; ++ } ++ jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); ++ jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); ++ ++ // invoke jvm static function to get results as Object[] ++ jobject jni_plan = (jobject)env->CallStaticObjectMethod( ++ graph_planner_clz_, graph_planner_method_id_, param1, param2); ++ ++ if (env->ExceptionCheck()) ++ { ++ env->ExceptionDescribe(); ++ env->ExceptionClear(); ++ std::cerr << "Error in calling GraphPlanner." << std::endl; ++ return plan; ++ } + +- if (env->ExceptionCheck()) { +- env->ExceptionDescribe(); +- env->ExceptionClear(); +- std::cerr << "Error in calling GraphPlanner." << std::endl; +- return plan; +- } ++ jmethodID method1 = env->GetMethodID( ++ env->GetObjectClass(jni_plan), "getPhysicalBytes", "()[B"); ++ jmethodID method2 = env->GetMethodID( ++ env->GetObjectClass(jni_plan), "getResultSchemaYaml", "()Ljava/lang/String;"); ++ ++ // 0-th object is the physical plan in byte array ++ jbyteArray res1 = (jbyteArray)env->CallObjectMethod(jni_plan, method1); ++ // 1-th object is the result schema in yaml format ++ jstring res2 = (jstring)env->CallObjectMethod(jni_plan, method2); ++ ++ if (res1 == NULL || res2 == NULL) ++ { ++ std::cerr << "Fail to generate plan." << std::endl; ++ return plan; ++ } ++ jbyte *str = env->GetByteArrayElements(res1, NULL); ++ jsize len = env->GetArrayLength(res1); ++ std::cout << "Physical plan size: " << len; + +- // 0-th object is the physical plan in byte array +- jbyteArray res = (jbyteArray)env->GetObjectArrayElement(resultArray, 0); +- // 1-th object is the result schema in yaml format +- jstring res2 = (jstring)env->GetObjectArrayElement(resultArray, 1); ++ plan.physical_plan.ParseFromArray(str, len); ++ plan.result_schema = env->GetStringUTFChars(res2, NULL); ++ ++ env->ReleaseByteArrayElements(res1, str, 0); ++ env->DeleteLocalRef(param1); ++ env->DeleteLocalRef(param2); ++ env->DeleteLocalRef(res1); ++ // remove new added jni objects ++ env->DeleteLocalRef(res2); ++ env->DeleteLocalRef(jni_plan); + +- if (res == NULL || res2 == NULL) { +- std::cerr << "Fail to generate plan." << std::endl; + return plan; + } +- jbyte *str = env->GetByteArrayElements(res, NULL); +- jsize len = env->GetArrayLength(res); +- std::cout << "Physical plan size: " << len; +- +- plan.physical_plan.ParseFromArray(str, len); +- plan.result_schema = env->GetStringUTFChars(res2, NULL); +- +- env->ReleaseByteArrayElements(res, str, 0); +- env->DeleteLocalRef(param1); +- env->DeleteLocalRef(param2); +- env->DeleteLocalRef(res); +- // remove new added jni objects +- env->DeleteLocalRef(res2); +- env->DeleteLocalRef(resultArray); +- +- return plan; +-} + #endif + + #if (!GRAPH_PLANNER_JNI_INVOKER) + +-void write_query_to_pipe(const std::string &path, +- const std::string &query_str) { +- std::cout << "write_query_to_pipe: " << path << std::endl; +- +- // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); +- int fd_to_java = open(path.c_str(), O_WRONLY); +- if (fd_to_java < 0) { +- std::cerr << "Fail to open pipe: " << path << std::endl; +- return; ++ void write_query_to_pipe(const std::string &path, ++ const std::string &query_str) ++ { ++ std::cout << "write_query_to_pipe: " << path << std::endl; ++ ++ // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); ++ int fd_to_java = open(path.c_str(), O_WRONLY); ++ if (fd_to_java < 0) ++ { ++ std::cerr << "Fail to open pipe: " << path << std::endl; ++ return; ++ } ++ std::cout << "open pipe done" << std::endl; ++ auto len = write(fd_to_java, query_str.c_str(), query_str.size()); ++ if (len != (int)query_str.size()) ++ { ++ std::cerr << "Fail to write query to pipe:" << len << std::endl; ++ return; ++ } ++ std::cout << "write_query_to_pipe done: " << len << std::endl; ++ close(fd_to_java); + } +- std::cout << "open pipe done" << std::endl; +- auto len = write(fd_to_java, query_str.c_str(), query_str.size()); +- if (len != (int)query_str.size()) { +- std::cerr << "Fail to write query to pipe:" << len << std::endl; +- return; ++ ++ void write_query_to_file(const std::string &path, ++ const std::string &query_str) ++ { ++ std::ofstream query_file(path); ++ query_file << query_str; ++ query_file.close(); + } +- std::cout << "write_query_to_pipe done: " << len << std::endl; +- close(fd_to_java); +-} +- +-void write_query_to_file(const std::string &path, +- const std::string &query_str) { +- std::ofstream query_file(path); +- query_file << query_str; +- query_file.close(); +-} +- +-physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) { +- std::cout << "plan str size: " << plan_str.size() << std::endl; +- physical::PhysicalPlan plan; +- if (!plan.ParseFromString(plan_str)) { +- std::cerr << "Fail to parse physical plan." << std::endl; +- return physical::PhysicalPlan(); ++ ++ physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) ++ { ++ std::cout << "plan str size: " << plan_str.size() << std::endl; ++ physical::PhysicalPlan plan; ++ if (!plan.ParseFromString(plan_str)) ++ { ++ std::cerr << "Fail to parse physical plan." << std::endl; ++ return physical::PhysicalPlan(); ++ } ++ return plan; + } +- return plan; +-} +- +-physical::PhysicalPlan +-compilePlanSubprocess(const std::string &class_path, +- const std::string &jna_path, +- const std::string &graph_schema_yaml, +- const std::string &graph_statistic_json, +- const std::string &compiler_config_path, +- const std::string &cypher_query_string) { +- physical::PhysicalPlan physical_plan; +- auto random_prefix = std::to_string( +- std::chrono::system_clock::now().time_since_epoch().count()); +- std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; +- std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; +- std::cout << "dst_query_path: " << dst_query_path +- << " dst_output_file: " << dst_output_file << std::endl; +- mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); +- mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); +- +- pid_t pid = fork(); +- +- if (pid == 0) { +- const char *const command_string_array[] = {"java", +- "-cp", +- class_path.c_str(), +- jna_path.c_str(), +- graph_schema_yaml.c_str(), +- graph_statistic_json.c_str(), +- GRAPH_PLANNER_FULL_NAME, +- compiler_config_path.c_str(), +- dst_query_path.c_str(), +- dst_output_file.c_str(), +- "/tmp/temp.cypher.yaml", +- NULL}; +- execvp(command_string_array[0], +- const_cast(command_string_array)); +- } else if (pid < 0) { +- std::cerr << "Error in fork." << std::endl; +- } else { +- write_query_to_pipe(dst_query_path, cypher_query_string); +- +- int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); +- if (fd_from_java < 0) { +- std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; +- return physical_plan; ++ ++ physical::PhysicalPlan ++ compilePlanSubprocess(const std::string &class_path, ++ const std::string &jna_path, ++ const std::string &graph_schema_yaml, ++ const std::string &graph_statistic_json, ++ const std::string &compiler_config_path, ++ const std::string &cypher_query_string) ++ { ++ physical::PhysicalPlan physical_plan; ++ auto random_prefix = std::to_string( ++ std::chrono::system_clock::now().time_since_epoch().count()); ++ std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; ++ std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; ++ std::cout << "dst_query_path: " << dst_query_path ++ << " dst_output_file: " << dst_output_file << std::endl; ++ mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); ++ mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); ++ ++ pid_t pid = fork(); ++ ++ if (pid == 0) ++ { ++ const char *const command_string_array[] = {"java", ++ "-cp", ++ class_path.c_str(), ++ jna_path.c_str(), ++ graph_schema_yaml.c_str(), ++ graph_statistic_json.c_str(), ++ GRAPH_PLANNER_FULL_NAME, ++ compiler_config_path.c_str(), ++ dst_query_path.c_str(), ++ dst_output_file.c_str(), ++ "/tmp/temp.cypher.yaml", ++ NULL}; ++ execvp(command_string_array[0], ++ const_cast(command_string_array)); + } +- std::vector stored_buffer; +- char buffer[128]; +- while (true) { +- ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); +- if (bytesRead <= 0) { +- break; +- } +- stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); ++ else if (pid < 0) ++ { ++ std::cerr << "Error in fork." << std::endl; + } +- physical_plan = readPhysicalPlan( +- std::string(stored_buffer.begin(), stored_buffer.end())); +- close(fd_from_java); +- +- int status; +- waitpid(pid, &status, 0); +- if (status != 0) { +- std::cerr << "Error in running command." << std::endl; ++ else ++ { ++ write_query_to_pipe(dst_query_path, cypher_query_string); ++ ++ int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); ++ if (fd_from_java < 0) ++ { ++ std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; ++ return physical_plan; ++ } ++ std::vector stored_buffer; ++ char buffer[128]; ++ while (true) ++ { ++ ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); ++ if (bytesRead <= 0) ++ { ++ break; ++ } ++ stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); ++ } ++ physical_plan = readPhysicalPlan( ++ std::string(stored_buffer.begin(), stored_buffer.end())); ++ close(fd_from_java); ++ ++ int status; ++ waitpid(pid, &status, 0); ++ if (status != 0) ++ { ++ std::cerr << "Error in running command." << std::endl; ++ } + } ++ unlink(dst_query_path.c_str()); ++ unlink(dst_output_file.c_str()); ++ return physical_plan; + } +- unlink(dst_query_path.c_str()); +- unlink(dst_output_file.c_str()); +- return physical_plan; +-} + #endif + +-Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, +- const std::string &cypher_query_string) { ++ Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, ++ const std::string &cypher_query_string) ++ { + #if (GRAPH_PLANNER_JNI_INVOKER) +- return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, +- jni_wrapper_.env(), compiler_config_path, +- cypher_query_string); ++ return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, ++ jni_wrapper_.env(), compiler_config_path, ++ cypher_query_string); + #else +- return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, +- graph_statistic_json_, compiler_config_path, +- cypher_query_string); ++ return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, ++ graph_statistic_json_, compiler_config_path, ++ cypher_query_string); + #endif +-} ++ } + + } // namespace gs +diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h +index 58ef9c7797..5c1229ef29 100644 +--- a/interactive_engine/compiler/src/main/native/graph_planner.h ++++ b/interactive_engine/compiler/src/main/native/graph_planner.h +@@ -29,119 +29,131 @@ limitations under the License. + #define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess + #endif + +-namespace gs { ++namespace gs ++{ + +-struct Plan { +- physical::PhysicalPlan physical_plan; +- std::string result_schema; +-}; ++ struct Plan ++ { ++ physical::PhysicalPlan physical_plan; ++ std::string result_schema; ++ }; + + #if (GRAPH_PLANNER_JNI_INVOKER) +-namespace jni { +-struct JNIEnvMark { +- JNIEnv *_env; +- +- JNIEnvMark(); +- JNIEnvMark(const std::string &jvm_options); +- ~JNIEnvMark(); +- JNIEnv *env(); +-}; +- +-} // namespace jni ++ namespace jni ++ { ++ struct JNIEnvMark ++ { ++ JNIEnv *_env; ++ ++ JNIEnvMark(); ++ JNIEnvMark(const std::string &jvm_options); ++ ~JNIEnvMark(); ++ JNIEnv *env(); ++ }; ++ ++ } // namespace jni + #endif + +-class GraphPlannerWrapper { +-public: +- static constexpr const char *kGraphPlannerClass = +- "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; +- static constexpr const char *GRAPH_PLANNER_FULL_NAME = +- "com.alibaba.graphscope.common.ir.tools.GraphPlanner"; +- static constexpr const char *kGraphPlannerMethod = "generatePhysicalPlan"; +- static constexpr const char *kGraphPlannerMethodSignature = +- "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; +- +- GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, +- const std::string &graph_schema_yaml, +- const std::string &graph_statistic_json = "") ++ class GraphPlannerWrapper ++ { ++ public: ++ static constexpr const char *kGraphPlannerClass = ++ "com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI"; ++ static constexpr const char *GRAPH_PLANNER_FULL_NAME = ++ "com.alibaba.graphscope.common.jna._native.GraphPlannerJNI"; ++ static constexpr const char *kGraphPlannerMethod = "compilePlan"; ++ static constexpr const char *kGraphPlannerMethodSignature = ++ "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; ++ ++ GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, ++ const std::string &graph_schema_yaml, ++ const std::string &graph_statistic_json = "") + #if (GRAPH_PLANNER_JNI_INVOKER) +- : jni_wrapper_(generate_jvm_options( +- java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { +- jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); +- if (clz == NULL) { +- std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; +- return; ++ : jni_wrapper_(generate_jvm_options( ++ java_path, jna_path, graph_schema_yaml, graph_statistic_json)) ++ { ++ jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); ++ if (clz == NULL) ++ { ++ std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; ++ return; ++ } ++ graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); ++ jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( ++ graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); ++ if (j_method_id == NULL) ++ { ++ std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; ++ return; ++ } ++ graph_planner_method_id_ = j_method_id; + } +- graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); +- jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( +- graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); +- if (j_method_id == NULL) { +- std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; +- return; +- } +- graph_planner_method_id_ = j_method_id; +- } + #else +- : jna_path_("-Djna.library.path=" + jna_path), +- graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), +- graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { +- class_path_ = expand_directory(java_path); +- } ++ : jna_path_("-Djna.library.path=" + jna_path), ++ graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), ++ graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) ++ { ++ class_path_ = expand_directory(java_path); ++ } + #endif + +- ~GraphPlannerWrapper() { ++ ~GraphPlannerWrapper() ++ { + #if (GRAPH_PLANNER_JNI_INVOKER) +- if (graph_planner_clz_ != NULL) { +- jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); +- } ++ if (graph_planner_clz_ != NULL) ++ { ++ jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); ++ } + #endif +- } ++ } + +- inline bool is_valid() { ++ inline bool is_valid() ++ { + #if (GRAPH_PLANNER_JNI_INVOKER) +- return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; ++ return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; + #else +- return true; // just return true, since we don't have a way to check the +- // validity when calling via subprocess. ++ return true; // just return true, since we don't have a way to check the ++ // validity when calling via subprocess. + #endif +- } +- +- /** +- * @brief Invoker GraphPlanner to generate a physical plan from a cypher +- * query. +- * @param compiler_config_path The path of compiler config file. +- * @param cypher_query_string The cypher query string. +- * @return physical plan in string. +- */ +- Plan CompilePlan(const std::string &compiler_config_path, +- const std::string &cypher_query_string); +- +-private: +- std::string generate_jvm_options(const std::string java_path, +- const std::string &jna_path, +- const std::string &graph_schema_yaml, +- const std::string &graph_statistic_json); +- // physical::PhysicalPlan compilePlanJNI(const std::string& +- // compiler_config_path, +- // const std::string& +- // cypher_query_string); +- std::string expand_directory(const std::string &path); ++ } ++ ++ /** ++ * @brief Invoker GraphPlanner to generate a physical plan from a cypher ++ * query. ++ * @param compiler_config_path The path of compiler config file. ++ * @param cypher_query_string The cypher query string. ++ * @return physical plan in string. ++ */ ++ Plan CompilePlan(const std::string &compiler_config_path, ++ const std::string &cypher_query_string); ++ ++ private: ++ std::string generate_jvm_options(const std::string java_path, ++ const std::string &jna_path, ++ const std::string &graph_schema_yaml, ++ const std::string &graph_statistic_json); ++ // physical::PhysicalPlan compilePlanJNI(const std::string& ++ // compiler_config_path, ++ // const std::string& ++ // cypher_query_string); ++ std::string expand_directory(const std::string &path); + #if (GRAPH_PLANNER_JNI_INVOKER) +- // We need to list all files in the directory, if exists. +- // The reason why we need to list all files in the directory is that +- // java -Djava.class.path=dir/* (in jni, which we are using)will not load all +- // jar files in the directory, While java -cp dir/* will load all jar files in +- // the directory. +- +- gs::jni::JNIEnvMark jni_wrapper_; +- jclass graph_planner_clz_; +- jmethodID graph_planner_method_id_; ++ // We need to list all files in the directory, if exists. ++ // The reason why we need to list all files in the directory is that ++ // java -Djava.class.path=dir/* (in jni, which we are using)will not load all ++ // jar files in the directory, While java -cp dir/* will load all jar files in ++ // the directory. ++ ++ gs::jni::JNIEnvMark jni_wrapper_; ++ jclass graph_planner_clz_; ++ jmethodID graph_planner_method_id_; + #else +- std::string class_path_; +- std::string jna_path_; +- std::string graph_schema_yaml_; +- std::string graph_statistic_json_; ++ std::string class_path_; ++ std::string jna_path_; ++ std::string graph_schema_yaml_; ++ std::string graph_statistic_json_; + #endif +-}; ++ }; + } // namespace gs + + #endif // PLANNER_GRAPH_PLANNER_H_ +diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +index 5469e8ad55..267363aa8a 100644 +--- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java ++++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +@@ -18,21 +18,21 @@ + + package com.alibaba.graphscope.common.ir; + +-import com.alibaba.graphscope.common.ir.tools.GraphPlanner; ++import com.alibaba.graphscope.common.jna._native.GraphPlannerJNI; ++import com.alibaba.graphscope.common.jna._native.JNIPlan; + import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +- + import org.junit.Test; + + public class JNITest { + @Test + public void test() throws Exception { +- Object[] objects = +- GraphPlanner.generatePhysicalPlan( ++ JNIPlan objects = ++ GraphPlannerJNI.compilePlan( + "conf/ir.compiler.properties", "Match (n) Return n, count(n)"); + GraphAlgebraPhysical.PhysicalPlan plan = +- GraphAlgebraPhysical.PhysicalPlan.parseFrom((byte[]) objects[0]); ++ GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); + System.out.println(plan); +- String resultSchema = (String) objects[1]; ++ String resultSchema = objects.resultSchemaYaml; + System.out.println(resultSchema); + } + } diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index df04307d853c..b93fa160bdc7 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -224,36 +224,6 @@ - - graph-planner-jni - - false - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - make - - run - - compile - - - - - - - - - - - - - @@ -324,6 +294,7 @@ build-helper-maven-plugin + generate-sources generate-sources add-source diff --git a/interactive_engine/compiler/src/main/cpp/README.md b/interactive_engine/compiler/src/main/cpp/README.md deleted file mode 100644 index cae277f0e1f3..000000000000 --- a/interactive_engine/compiler/src/main/cpp/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# GraphScope Interactive Query Planner - -This directory contains the c++ wrapper of Interactive Query Planner, which is implemented in [java](https://github.com/alibaba/GraphScope/tree/main/interactive_engine/compiler). \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.cc b/interactive_engine/compiler/src/main/cpp/graph_planner.cc deleted file mode 100644 index 417169a1301b..000000000000 --- a/interactive_engine/compiler/src/main/cpp/graph_planner.cc +++ /dev/null @@ -1,386 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -#include -#include -#include -#include -#include -#include - -#include "graph_planner.h" -#include - -namespace gs { -#if (GRAPH_PLANNER_JNI_INVOKER) -namespace jni { - -static JavaVM *_jvm = NULL; - -JavaVM *CreateJavaVM(const std::string &jvm_options) { - const char *p, *q; - const char *jvm_opts; - if (jvm_options.empty()) { - jvm_opts = getenv("FLEX_JVM_OPTS"); - } else { - jvm_opts = jvm_options.c_str(); - } - if (jvm_opts == NULL) { - std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; - return NULL; - } - std::cout << "Jvm opts str: " << jvm_opts << std::endl; - - if (*jvm_opts == '\0') - return NULL; - - int num_of_opts = 1; - for (const char *p = jvm_opts; *p; p++) { - if (*p == ' ') - num_of_opts++; - } - - if (num_of_opts == 0) - return NULL; - - JavaVM *jvm = NULL; - JNIEnv *env = NULL; - int i = 0; - int status = 1; - JavaVMInitArgs vm_args; - - JavaVMOption *options = new JavaVMOption[num_of_opts]; - memset(options, 0, sizeof(JavaVMOption) * num_of_opts); - - for (p = q = jvm_opts;; p++) { - if (*p == ' ' || *p == '\0') { - if (q >= p) { - goto ret; - } - char *opt = new char[p - q + 1]; - memcpy(opt, q, p - q); - opt[p - q] = '\0'; - options[i++].optionString = opt; - q = p + 1; // assume opts are separated by single space - if (*p == '\0') - break; - } - } - - memset(&vm_args, 0, sizeof(vm_args)); - vm_args.version = JNI_VERSION_1_8; - vm_args.nOptions = num_of_opts; - vm_args.options = options; - - status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); - if (status == JNI_OK) { - std::cout << "Create java virtual machine successfully." << std::endl; - } else if (status == JNI_EEXIST) { - std::cout << "JNI evn already exists." << std::endl; - } else { - std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" - << status << ")" << std::endl; - } - -ret: - for (int i = 0; i < num_of_opts; i++) { - delete[] options[i].optionString; - } - delete[] options; - return jvm; -} - -// One process can only create jvm for once. -JavaVM *GetJavaVM(const std::string jvm_options = "") { - if (_jvm == NULL) { - // Try to find whether there exists one javaVM - jsize nVMs; - JNI_GetCreatedJavaVMs(NULL, 0, - &nVMs); // 1. just get the required array length - std::cout << "Found " << nVMs << " VMs existing in this process." - << std::endl; - JavaVM **buffer = new JavaVM *[nVMs]; - JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data - for (auto i = 0; i < nVMs; ++i) { - if (buffer[i] != NULL) { - _jvm = buffer[i]; - std::cout << "Found index " << i << " VM non null " - << reinterpret_cast(_jvm) << std::endl; - return _jvm; - } - } - _jvm = CreateJavaVM(jvm_options); - std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; - } - return _jvm; -} - -JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} - -JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) { - if (!GetJavaVM(jvm_options)) { - return; - } - int status = - GetJavaVM(jvm_options) - ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); - if (status != JNI_OK) { - std::cerr << "Error attach current thread: " << status << std::endl; - } -} - -JNIEnvMark::~JNIEnvMark() { - if (_env) { - GetJavaVM()->DetachCurrentThread(); - } -} - -JNIEnv *JNIEnvMark::env() { return _env; } - -} // namespace jni - -#endif - -std::vector list_files(const std::string &path) { - // list all files in the directory - std::vector files; - for (const auto &entry : std::filesystem::directory_iterator(path)) { - files.push_back(entry.path().string()); - } - return files; -} - -void iterate_over_director(const std::string& dir_or_path, std::vector& output_paths){ - if (dir_or_path.empty()) { - return; - } - if (std::filesystem::is_directory(dir_or_path)) { - auto files = list_files(dir_or_path); - output_paths.insert(output_paths.end(), files.begin(), files.end()); - } else { - output_paths.push_back(dir_or_path); - } -} - -std::string GraphPlannerWrapper::expand_directory(const std::string &path) { - std::vector paths; - std::string::size_type start = 0; - std::string::size_type end = path.find(':'); - while (end != std::string::npos) { - auto sub_path = path.substr(start, end - start); - iterate_over_director(sub_path, paths); - start = end + 1; - end = path.find(':', start); - } - auto sub_path = path.substr(start); - iterate_over_director(sub_path, paths); - std::stringstream ss; - for (const auto &p : paths) { - ss << p << ":"; - } - return ss.str(); -} - -#if (GRAPH_PLANNER_JNI_INVOKER) - -std::string GraphPlannerWrapper::generate_jvm_options( - const std::string java_path, const std::string &jna_path, - const std::string &graph_schema_yaml, - const std::string &graph_statistic_json) { - auto expanded_java_path = expand_directory(java_path); - std::cout << "Expanded java path: " << expanded_java_path << std::endl; - std::string jvm_options = "-Djava.class.path=" + expanded_java_path; - jvm_options += " -Djna.library.path=" + jna_path; - jvm_options += " -Dgraph.schema=" + graph_schema_yaml; - if (!graph_statistic_json.empty()) { - jvm_options += " -Dgraph.statistic=" + graph_statistic_json; - } - return jvm_options; -} - -Plan compilePlanJNI(jclass graph_planner_clz_, - jmethodID graph_planner_method_id_, JNIEnv *env, - const std::string &compiler_config_path, - const std::string &cypher_query_string) { - jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), - nullptr); - Plan plan; - if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { - std::cerr << "Invalid GraphPlannerWrapper." << std::endl; - return plan; - } - jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); - jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); - - // invoke jvm static function to get results as Object[] - jobjectArray resultArray = (jobjectArray)env->CallStaticObjectMethod( - graph_planner_clz_, graph_planner_method_id_, param1, param2); - - if (env->ExceptionCheck()) { - env->ExceptionDescribe(); - env->ExceptionClear(); - std::cerr << "Error in calling GraphPlanner." << std::endl; - return plan; - } - - // 0-th object is the physical plan in byte array - jbyteArray res = (jbyteArray)env->GetObjectArrayElement(resultArray, 0); - // 1-th object is the result schema in yaml format - jstring res2 = (jstring)env->GetObjectArrayElement(resultArray, 1); - - if (res == NULL || res2 == NULL) { - std::cerr << "Fail to generate plan." << std::endl; - return plan; - } - jbyte *str = env->GetByteArrayElements(res, NULL); - jsize len = env->GetArrayLength(res); - std::cout << "Physical plan size: " << len; - - plan.physical_plan.ParseFromArray(str, len); - plan.result_schema = env->GetStringUTFChars(res2, NULL); - - env->ReleaseByteArrayElements(res, str, 0); - env->DeleteLocalRef(param1); - env->DeleteLocalRef(param2); - env->DeleteLocalRef(res); - // remove new added jni objects - env->DeleteLocalRef(res2); - env->DeleteLocalRef(resultArray); - - return plan; -} -#endif - -#if (!GRAPH_PLANNER_JNI_INVOKER) - -void write_query_to_pipe(const std::string &path, - const std::string &query_str) { - std::cout << "write_query_to_pipe: " << path << std::endl; - - // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - int fd_to_java = open(path.c_str(), O_WRONLY); - if (fd_to_java < 0) { - std::cerr << "Fail to open pipe: " << path << std::endl; - return; - } - std::cout << "open pipe done" << std::endl; - auto len = write(fd_to_java, query_str.c_str(), query_str.size()); - if (len != (int)query_str.size()) { - std::cerr << "Fail to write query to pipe:" << len << std::endl; - return; - } - std::cout << "write_query_to_pipe done: " << len << std::endl; - close(fd_to_java); -} - -void write_query_to_file(const std::string &path, - const std::string &query_str) { - std::ofstream query_file(path); - query_file << query_str; - query_file.close(); -} - -physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) { - std::cout << "plan str size: " << plan_str.size() << std::endl; - physical::PhysicalPlan plan; - if (!plan.ParseFromString(plan_str)) { - std::cerr << "Fail to parse physical plan." << std::endl; - return physical::PhysicalPlan(); - } - return plan; -} - -physical::PhysicalPlan -compilePlanSubprocess(const std::string &class_path, - const std::string &jna_path, - const std::string &graph_schema_yaml, - const std::string &graph_statistic_json, - const std::string &compiler_config_path, - const std::string &cypher_query_string) { - physical::PhysicalPlan physical_plan; - auto random_prefix = std::to_string( - std::chrono::system_clock::now().time_since_epoch().count()); - std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; - std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; - std::cout << "dst_query_path: " << dst_query_path - << " dst_output_file: " << dst_output_file << std::endl; - mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); - - pid_t pid = fork(); - - if (pid == 0) { - const char *const command_string_array[] = {"java", - "-cp", - class_path.c_str(), - jna_path.c_str(), - graph_schema_yaml.c_str(), - graph_statistic_json.c_str(), - GRAPH_PLANNER_FULL_NAME, - compiler_config_path.c_str(), - dst_query_path.c_str(), - dst_output_file.c_str(), - "/tmp/temp.cypher.yaml", - NULL}; - execvp(command_string_array[0], - const_cast(command_string_array)); - } else if (pid < 0) { - std::cerr << "Error in fork." << std::endl; - } else { - write_query_to_pipe(dst_query_path, cypher_query_string); - - int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); - if (fd_from_java < 0) { - std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; - return physical_plan; - } - std::vector stored_buffer; - char buffer[128]; - while (true) { - ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); - if (bytesRead <= 0) { - break; - } - stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); - } - physical_plan = readPhysicalPlan( - std::string(stored_buffer.begin(), stored_buffer.end())); - close(fd_from_java); - - int status; - waitpid(pid, &status, 0); - if (status != 0) { - std::cerr << "Error in running command." << std::endl; - } - } - unlink(dst_query_path.c_str()); - unlink(dst_output_file.c_str()); - return physical_plan; -} -#endif - -Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, - const std::string &cypher_query_string) { -#if (GRAPH_PLANNER_JNI_INVOKER) - return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, - jni_wrapper_.env(), compiler_config_path, - cypher_query_string); -#else - return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, - graph_statistic_json_, compiler_config_path, - cypher_query_string); -#endif -} - -} // namespace gs diff --git a/interactive_engine/compiler/src/main/cpp/graph_planner.h b/interactive_engine/compiler/src/main/cpp/graph_planner.h deleted file mode 100644 index 58ef9c779788..000000000000 --- a/interactive_engine/compiler/src/main/cpp/graph_planner.h +++ /dev/null @@ -1,147 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#ifndef PLANNER_GRAPH_PLANNER_H_ -#define PLANNER_GRAPH_PLANNER_H_ - -#include -#include -#include -#include -#include -#include - -#include "flex/proto_generated_gie/physical.pb.h" - -#ifndef GRAPH_PLANNER_JNI_INVOKER -#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess -#endif - -namespace gs { - -struct Plan { - physical::PhysicalPlan physical_plan; - std::string result_schema; -}; - -#if (GRAPH_PLANNER_JNI_INVOKER) -namespace jni { -struct JNIEnvMark { - JNIEnv *_env; - - JNIEnvMark(); - JNIEnvMark(const std::string &jvm_options); - ~JNIEnvMark(); - JNIEnv *env(); -}; - -} // namespace jni -#endif - -class GraphPlannerWrapper { -public: - static constexpr const char *kGraphPlannerClass = - "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; - static constexpr const char *GRAPH_PLANNER_FULL_NAME = - "com.alibaba.graphscope.common.ir.tools.GraphPlanner"; - static constexpr const char *kGraphPlannerMethod = "generatePhysicalPlan"; - static constexpr const char *kGraphPlannerMethodSignature = - "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; - - GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, - const std::string &graph_schema_yaml, - const std::string &graph_statistic_json = "") -#if (GRAPH_PLANNER_JNI_INVOKER) - : jni_wrapper_(generate_jvm_options( - java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { - jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); - if (clz == NULL) { - std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; - return; - } - graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); - jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( - graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); - if (j_method_id == NULL) { - std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; - return; - } - graph_planner_method_id_ = j_method_id; - } -#else - : jna_path_("-Djna.library.path=" + jna_path), - graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), - graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { - class_path_ = expand_directory(java_path); - } -#endif - - ~GraphPlannerWrapper() { -#if (GRAPH_PLANNER_JNI_INVOKER) - if (graph_planner_clz_ != NULL) { - jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); - } -#endif - } - - inline bool is_valid() { -#if (GRAPH_PLANNER_JNI_INVOKER) - return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; -#else - return true; // just return true, since we don't have a way to check the - // validity when calling via subprocess. -#endif - } - - /** - * @brief Invoker GraphPlanner to generate a physical plan from a cypher - * query. - * @param compiler_config_path The path of compiler config file. - * @param cypher_query_string The cypher query string. - * @return physical plan in string. - */ - Plan CompilePlan(const std::string &compiler_config_path, - const std::string &cypher_query_string); - -private: - std::string generate_jvm_options(const std::string java_path, - const std::string &jna_path, - const std::string &graph_schema_yaml, - const std::string &graph_statistic_json); - // physical::PhysicalPlan compilePlanJNI(const std::string& - // compiler_config_path, - // const std::string& - // cypher_query_string); - std::string expand_directory(const std::string &path); -#if (GRAPH_PLANNER_JNI_INVOKER) - // We need to list all files in the directory, if exists. - // The reason why we need to list all files in the directory is that - // java -Djava.class.path=dir/* (in jni, which we are using)will not load all - // jar files in the directory, While java -cp dir/* will load all jar files in - // the directory. - - gs::jni::JNIEnvMark jni_wrapper_; - jclass graph_planner_clz_; - jmethodID graph_planner_method_id_; -#else - std::string class_path_; - std::string jna_path_; - std::string graph_schema_yaml_; - std::string graph_statistic_json_; -#endif -}; -} // namespace gs - -#endif // PLANNER_GRAPH_PLANNER_H_ diff --git a/interactive_engine/compiler/src/main/cpp/test/test.cc b/interactive_engine/compiler/src/main/cpp/test/test.cc deleted file mode 100644 index 2d28e884dfb9..000000000000 --- a/interactive_engine/compiler/src/main/cpp/test/test.cc +++ /dev/null @@ -1,83 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -#include "graph_planner.h" - -#include - -std::string get_dir_name() { - // Get the directory of this source file - std::string current_dir = __FILE__; - size_t pos = current_dir.find_last_of("/"); - current_dir = current_dir.substr(0, pos); - return current_dir; -} - -void check_path_exits(const std::string& path) { - // split path by ':' - std::vector paths; - std::string::size_type start = 0; - std::string::size_type end = path.find(':'); - while (end != std::string::npos) { - auto sub_path = path.substr(start, end - start); - paths.push_back(sub_path); - start = end + 1; - end = path.find(':', start); - } - auto sub_path = path.substr(start); - paths.push_back(sub_path); - - for (const auto& p : paths) { - struct stat buffer; - if (stat(p.c_str(), &buffer) != 0) { - std::cerr << "Path not exists: " << p << std::endl; - exit(1); - } - } - std::cout<< "Path exists: " << path << std::endl; -} - -int main(int argc, char **argv) { - auto current_dir = get_dir_name(); - std::string java_class_path = - current_dir + "/../../../../target/compiler-0.0.1-SNAPSHOT.jar"; - java_class_path+= ":" + current_dir + "/../../../../target/libs/"; - std::string jna_class_path = - current_dir + "/../../../../../executor/ir/target/release/"; - std::string graph_schema_yaml = - current_dir + - "/../../../../../../flex/interactive/examples/modern_graph/graph.yaml"; - std::string graph_statistic_json = - current_dir + - "/../../../test/resources/statistics/modern_statistics.json"; - - // check director or file exists - check_path_exits(java_class_path); - check_path_exits(jna_class_path); - check_path_exits(graph_schema_yaml); - check_path_exits(graph_statistic_json); - - gs::GraphPlannerWrapper graph_planner_wrapper( - java_class_path, jna_class_path, graph_schema_yaml, graph_statistic_json); - - std::string cypher_query_string = "MATCH (a:person) RETURN a.name"; - std::string config_path = - current_dir + - "/../../../../../../flex/tests/hqps/interactive_config_test.yaml"; - auto plan = - graph_planner_wrapper.CompilePlan(config_path, cypher_query_string); - std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; - std::cout << "schema: " << plan.result_schema << std::endl; - return 0; -} \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index ab11c9b198b2..9a28564c1812 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -210,8 +210,8 @@ private static GraphProperties getTestGraph(Configs configs) { case "rust-mcsr": testGraph = TestGraphFactory.RUST_MCSR; break; - case "cpp-mcsr": - logger.info("using cpp-mcsr as test graph"); + case "native-mcsr": + logger.info("using native-mcsr as test graph"); testGraph = TestGraphFactory.CPP_MCSR; break; default: diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index e55682a86cdb..1670362ccfcc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -78,7 +78,7 @@ public class YamlConfigs extends Configs { if (configs.get("compute_engine.store.type") != null) { return configs.get("compute_engine.store.type"); } else { - return "cpp-mcsr"; + return "native-mcsr"; } }) .put( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index 15b16dcbcf71..d2d9c1e329ee 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -349,7 +349,7 @@ public static class Config { // option configurations. public static final com.alibaba.graphscope.common.config.Config TYPE = com.alibaba.graphscope.common.config.Config.stringConfig( - "type", "UNKNOWN"); // cypher or cpp + "type", "UNKNOWN"); // cypher or native public static final com.alibaba.graphscope.common.config.Config QUERY = com.alibaba.graphscope.common.config.Config.stringConfig("query", "UNKNOWN"); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 991ae7b89d5f..77dc36d2a893 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -249,23 +249,6 @@ public static Summary generatePlan(String config_path, String query_string) thro return summary; } - public static Object[] generatePhysicalPlan(String config_path, String query_string) - throws Exception { - Summary summary = generatePlan(config_path, query_string); - LogicalPlan logicalPlan = summary.getLogicalPlan(); - PhysicalPlan physicalPlan = summary.physicalPlan; - Configs extraConfigs = createExtraConfigs(null); - StoredProcedureMeta procedureMeta = - new StoredProcedureMeta( - extraConfigs, - query_string, - logicalPlan.getOutputType(), - logicalPlan.getDynamicParams()); - ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); - StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); - return new Object[] {physicalPlan.getContent(), new String(metaStream.toByteArray())}; - } - public static void main(String[] args) throws Exception { if (args.length < 4 || args[0].isEmpty() diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java new file mode 100644 index 000000000000..3764242c3edc --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java @@ -0,0 +1,54 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.jna._native; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.google.common.collect.ImmutableMap; + +import java.io.ByteArrayOutputStream; + +public class GraphPlannerJNI { + /** + * Provide a java-side implementation to compile the query in string to a physical plan + * @param config + * @param query + * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, + * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, + * resultSchemaYaml defines the result specification of the query in yaml format + * @throws Exception + */ + public static JNIPlan compilePlan(String config, String query) throws Exception { + GraphPlanner.Summary summary = GraphPlanner.generatePlan(config, query); + LogicalPlan logicalPlan = summary.getLogicalPlan(); + PhysicalPlan physicalPlan = summary.getPhysicalPlan(); + StoredProcedureMeta procedureMeta = + new StoredProcedureMeta( + new Configs(ImmutableMap.of()), + query, + logicalPlan.getOutputType(), + logicalPlan.getDynamicParams()); + ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); + StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); + return new JNIPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java new file mode 100644 index 000000000000..689c9c74efc5 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java @@ -0,0 +1,37 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.jna._native; + +public class JNIPlan { + public final byte[] physicalBytes; + public final String resultSchemaYaml; + + public JNIPlan(byte[] physicalBytes, String resultSchemaYaml) { + this.physicalBytes = physicalBytes; + this.resultSchemaYaml = resultSchemaYaml; + } + + public byte[] getPhysicalBytes() { + return physicalBytes; + } + + public String getResultSchemaYaml() { + return resultSchemaYaml; + } +} diff --git a/interactive_engine/compiler/src/main/cpp/CMakeLists.txt b/interactive_engine/compiler/src/main/native/CMakeLists.txt similarity index 100% rename from interactive_engine/compiler/src/main/cpp/CMakeLists.txt rename to interactive_engine/compiler/src/main/native/CMakeLists.txt diff --git a/interactive_engine/compiler/src/main/native/README.md b/interactive_engine/compiler/src/main/native/README.md new file mode 100644 index 000000000000..ca814ae2d4bf --- /dev/null +++ b/interactive_engine/compiler/src/main/native/README.md @@ -0,0 +1,85 @@ +# Using the Graph Planner JNI Interface for C++ Invocation + +Follow the steps below to get started with the Graph Planner JNI interface for c++ invocation. + +## Getting Started +### Step 1: Build the Project + +Navigate to the project directory and build the package using Maven: +```bash +cd interactive_engine +mvn clean package -DskipTests -Pgraph-planner-jni +``` + +### Step 2: Locate and Extract the Package + +After the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents of the tarball: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +### Step 3: Run the Example Binary + +To demonstrate the usage of the JNI interface, an example binary test_graph_planner is provided. Use the following command to execute it: + +```bash +# bin/test_graph_planner +bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/interactive_config_test.yaml +``` + +The output includes the physical plan and result schema in YAML format. Below is an example of a result schema: + +```yaml +schema: + name: default + description: default desc + mode: READ + extension: .so + library: libdefault.so + params: [] +returns: + - name: n + type: {primitive_type: DT_UNKNOWN} + - name: $f1 + type: {primitive_type: DT_SIGNED_INT64} +type: UNKNOWN +query: MATCH (n) RETURN n, COUNT(n); +``` + +The `returns` field defines the result schema. Each entry’s name specifies the column name, and the order of each entry determines the column IDs. + +## Explanation + +Below is a brief explanation of the interface provided by the example: + +### Constructor + +```cpp +/** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format + * @param graph_statistic_json Path to the graph statistics file in JSON format + */ +GraphPlannerWrapper(const std::string &java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json = ""); +``` + +## Method + +```cpp +/** + * @brief Compiles a Cypher query into a physical plan via JNI invocation. + * @param compiler_config_path Path to the compiler configuration file. + * @param cypher_query_string Cypher query string to compile. + * @return Physical plan in bytes and result schema in YAML format. + */ +Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string); +``` diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc new file mode 100644 index 000000000000..69da83c9c8ef --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.cc @@ -0,0 +1,456 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include +#include +#include +#include +#include +#include + +#include "graph_planner.h" +#include + +namespace gs +{ +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + + static JavaVM *_jvm = NULL; + + JavaVM *CreateJavaVM(const std::string &jvm_options) + { + const char *p, *q; + const char *jvm_opts; + if (jvm_options.empty()) + { + jvm_opts = getenv("FLEX_JVM_OPTS"); + } + else + { + jvm_opts = jvm_options.c_str(); + } + if (jvm_opts == NULL) + { + std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; + return NULL; + } + std::cout << "Jvm opts str: " << jvm_opts << std::endl; + + if (*jvm_opts == '\0') + return NULL; + + int num_of_opts = 1; + for (const char *p = jvm_opts; *p; p++) + { + if (*p == ' ') + num_of_opts++; + } + + if (num_of_opts == 0) + return NULL; + + JavaVM *jvm = NULL; + JNIEnv *env = NULL; + int i = 0; + int status = 1; + JavaVMInitArgs vm_args; + + JavaVMOption *options = new JavaVMOption[num_of_opts]; + memset(options, 0, sizeof(JavaVMOption) * num_of_opts); + + for (p = q = jvm_opts;; p++) + { + if (*p == ' ' || *p == '\0') + { + if (q >= p) + { + goto ret; + } + char *opt = new char[p - q + 1]; + memcpy(opt, q, p - q); + opt[p - q] = '\0'; + options[i++].optionString = opt; + q = p + 1; // assume opts are separated by single space + if (*p == '\0') + break; + } + } + + memset(&vm_args, 0, sizeof(vm_args)); + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = num_of_opts; + vm_args.options = options; + + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + if (status == JNI_OK) + { + std::cout << "Create java virtual machine successfully." << std::endl; + } + else if (status == JNI_EEXIST) + { + std::cout << "JNI evn already exists." << std::endl; + } + else + { + std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")" << std::endl; + } + + ret: + for (int i = 0; i < num_of_opts; i++) + { + delete[] options[i].optionString; + } + delete[] options; + return jvm; + } + + // One process can only create jvm for once. + JavaVM *GetJavaVM(const std::string jvm_options = "") + { + if (_jvm == NULL) + { + // Try to find whether there exists one javaVM + jsize nVMs; + JNI_GetCreatedJavaVMs(NULL, 0, + &nVMs); // 1. just get the required array length + std::cout << "Found " << nVMs << " VMs existing in this process." + << std::endl; + JavaVM **buffer = new JavaVM *[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + for (auto i = 0; i < nVMs; ++i) + { + if (buffer[i] != NULL) + { + _jvm = buffer[i]; + std::cout << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm) << std::endl; + return _jvm; + } + } + _jvm = CreateJavaVM(jvm_options); + std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; + } + return _jvm; + } + + JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + + JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) + { + if (!GetJavaVM(jvm_options)) + { + return; + } + int status = + GetJavaVM(jvm_options) + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + if (status != JNI_OK) + { + std::cerr << "Error attach current thread: " << status << std::endl; + } + } + + JNIEnvMark::~JNIEnvMark() + { + if (_env) + { + GetJavaVM()->DetachCurrentThread(); + } + } + + JNIEnv *JNIEnvMark::env() { return _env; } + + } // namespace jni + +#endif + + std::vector list_files(const std::string &path) + { + // list all files in the directory + std::vector files; + for (const auto &entry : std::filesystem::directory_iterator(path)) + { + files.push_back(entry.path().string()); + } + return files; + } + + void iterate_over_director(const std::string &dir_or_path, std::vector &output_paths) + { + if (dir_or_path.empty()) + { + return; + } + if (std::filesystem::is_directory(dir_or_path)) + { + auto files = list_files(dir_or_path); + output_paths.insert(output_paths.end(), files.begin(), files.end()); + } + else + { + output_paths.push_back(dir_or_path); + } + } + + std::string GraphPlannerWrapper::expand_directory(const std::string &path) + { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + iterate_over_director(sub_path, paths); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + iterate_over_director(sub_path, paths); + std::stringstream ss; + for (const auto &p : paths) + { + ss << p << ":"; + } + return ss.str(); + } + +#if (GRAPH_PLANNER_JNI_INVOKER) + + std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { + auto expanded_java_path = expand_directory(java_path); + std::cout << "Expanded java path: " << expanded_java_path << std::endl; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + if (!graph_statistic_json.empty()) + { + jvm_options += " -Dgraph.statistics=" + graph_statistic_json; + } + return jvm_options; + } + + Plan compilePlanJNI(jclass graph_planner_clz_, + jmethodID graph_planner_method_id_, JNIEnv *env, + const std::string &compiler_config_path, + const std::string &cypher_query_string) + { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + nullptr); + Plan plan; + if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) + { + std::cerr << "Invalid GraphPlannerWrapper." << std::endl; + return plan; + } + jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + + // invoke jvm static function to get results as Object[] + jobject jni_plan = (jobject)env->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2); + + if (env->ExceptionCheck()) + { + env->ExceptionDescribe(); + env->ExceptionClear(); + std::cerr << "Error in calling GraphPlanner." << std::endl; + return plan; + } + + jmethodID method1 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getPhysicalBytes", "()[B"); + jmethodID method2 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getResultSchemaYaml", "()Ljava/lang/String;"); + + // 0-th object is the physical plan in byte array + jbyteArray res1 = (jbyteArray)env->CallObjectMethod(jni_plan, method1); + // 1-th object is the result schema in yaml format + jstring res2 = (jstring)env->CallObjectMethod(jni_plan, method2); + + if (res1 == NULL || res2 == NULL) + { + std::cerr << "Fail to generate plan." << std::endl; + return plan; + } + jbyte *str = env->GetByteArrayElements(res1, NULL); + jsize len = env->GetArrayLength(res1); + std::cout << "Physical plan size: " << len; + + plan.physical_plan.ParseFromArray(str, len); + plan.result_schema = env->GetStringUTFChars(res2, NULL); + + env->ReleaseByteArrayElements(res1, str, 0); + env->DeleteLocalRef(param1); + env->DeleteLocalRef(param2); + env->DeleteLocalRef(res1); + // remove new added jni objects + env->DeleteLocalRef(res2); + env->DeleteLocalRef(jni_plan); + + return plan; + } +#endif + +#if (!GRAPH_PLANNER_JNI_INVOKER) + + void write_query_to_pipe(const std::string &path, + const std::string &query_str) + { + std::cout << "write_query_to_pipe: " << path << std::endl; + + // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + int fd_to_java = open(path.c_str(), O_WRONLY); + if (fd_to_java < 0) + { + std::cerr << "Fail to open pipe: " << path << std::endl; + return; + } + std::cout << "open pipe done" << std::endl; + auto len = write(fd_to_java, query_str.c_str(), query_str.size()); + if (len != (int)query_str.size()) + { + std::cerr << "Fail to write query to pipe:" << len << std::endl; + return; + } + std::cout << "write_query_to_pipe done: " << len << std::endl; + close(fd_to_java); + } + + void write_query_to_file(const std::string &path, + const std::string &query_str) + { + std::ofstream query_file(path); + query_file << query_str; + query_file.close(); + } + + physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) + { + std::cout << "plan str size: " << plan_str.size() << std::endl; + physical::PhysicalPlan plan; + if (!plan.ParseFromString(plan_str)) + { + std::cerr << "Fail to parse physical plan." << std::endl; + return physical::PhysicalPlan(); + } + return plan; + } + + physical::PhysicalPlan + compilePlanSubprocess(const std::string &class_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json, + const std::string &compiler_config_path, + const std::string &cypher_query_string) + { + physical::PhysicalPlan physical_plan; + auto random_prefix = std::to_string( + std::chrono::system_clock::now().time_since_epoch().count()); + std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; + std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; + std::cout << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file << std::endl; + mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + + pid_t pid = fork(); + + if (pid == 0) + { + const char *const command_string_array[] = {"java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + GRAPH_PLANNER_FULL_NAME, + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; + execvp(command_string_array[0], + const_cast(command_string_array)); + } + else if (pid < 0) + { + std::cerr << "Error in fork." << std::endl; + } + else + { + write_query_to_pipe(dst_query_path, cypher_query_string); + + int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); + if (fd_from_java < 0) + { + std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; + return physical_plan; + } + std::vector stored_buffer; + char buffer[128]; + while (true) + { + ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); + if (bytesRead <= 0) + { + break; + } + stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); + } + physical_plan = readPhysicalPlan( + std::string(stored_buffer.begin(), stored_buffer.end())); + close(fd_from_java); + + int status; + waitpid(pid, &status, 0); + if (status != 0) + { + std::cerr << "Error in running command." << std::endl; + } + } + unlink(dst_query_path.c_str()); + unlink(dst_output_file.c_str()); + return physical_plan; + } +#endif + + /** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return The physical plan in bytes and result schema in yaml. + */ + Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string) + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, + jni_wrapper_.env(), compiler_config_path, + cypher_query_string); +#else + return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, + graph_statistic_json_, compiler_config_path, + cypher_query_string); +#endif + } + +} // namespace gs diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h new file mode 100644 index 000000000000..28ce4f15b0cd --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.h @@ -0,0 +1,167 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#ifndef PLANNER_GRAPH_PLANNER_H_ +#define PLANNER_GRAPH_PLANNER_H_ + +#include +#include +#include +#include +#include +#include + +#include "flex/proto_generated_gie/physical.pb.h" + +#ifndef GRAPH_PLANNER_JNI_INVOKER +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess +#endif + +namespace gs +{ + + struct Plan + { + physical::PhysicalPlan physical_plan; + std::string result_schema; + }; + +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + struct JNIEnvMark + { + JNIEnv *_env; + + JNIEnvMark(); + JNIEnvMark(const std::string &jvm_options); + ~JNIEnvMark(); + JNIEnv *env(); + }; + + } // namespace jni +#endif + + class GraphPlannerWrapper + { + public: + static constexpr const char *kGraphPlannerClass = + "com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI"; + static constexpr const char *GRAPH_PLANNER_FULL_NAME = + "com.alibaba.graphscope.common.jna._native.GraphPlannerJNI"; + static constexpr const char *kGraphPlannerMethod = "compilePlan"; + static constexpr const char *kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/common/jna/_native/JNIPlan;"; + + /** + * @brief Construct a new Graph Planner Wrapper object + * @param java_path java class path + * @param jna_path jna library path + * @param graph_schema_yaml graph schema path + * @param graph_statistic_json graph statistic path + */ + GraphPlannerWrapper(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json = "") +#if (GRAPH_PLANNER_JNI_INVOKER) + : jni_wrapper_(generate_jvm_options( + java_path, jna_path, graph_schema_yaml, graph_statistic_json)) + { + jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); + if (clz == NULL) + { + std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; + return; + } + graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); + jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( + graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); + if (j_method_id == NULL) + { + std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; + return; + } + graph_planner_method_id_ = j_method_id; + } +#else + : jna_path_("-Djna.library.path=" + jna_path), + graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), + graph_statistic_json_("-Dgraph.statistics=" + graph_statistic_json) + { + class_path_ = expand_directory(java_path); + } +#endif + + ~GraphPlannerWrapper() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + if (graph_planner_clz_ != NULL) + { + jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); + } +#endif + } + + inline bool is_valid() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; +#else + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. +#endif + } + + /** + * @brief Invoker GraphPlanner to generate a physical plan from a cypher + * query. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return physical plan in string. + */ + Plan CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string); + + private: + std::string generate_jvm_options(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); + // physical::PhysicalPlan compilePlanJNI(const std::string& + // compiler_config_path, + // const std::string& + // cypher_query_string); + std::string expand_directory(const std::string &path); +#if (GRAPH_PLANNER_JNI_INVOKER) + // We need to list all files in the directory, if exists. + // The reason why we need to list all files in the directory is that + // java -Djava.class.path=dir/* (in jni, which we are using)will not load all + // jar files in the directory, While java -cp dir/* will load all jar files in + // the directory. + + gs::jni::JNIEnvMark jni_wrapper_; + jclass graph_planner_clz_; + jmethodID graph_planner_method_id_; +#else + std::string class_path_; + std::string jna_path_; + std::string graph_schema_yaml_; + std::string graph_statistic_json_; +#endif + }; +} // namespace gs + +#endif // PLANNER_GRAPH_PLANNER_H_ diff --git a/interactive_engine/compiler/src/main/native/test/test.cc b/interactive_engine/compiler/src/main/native/test/test.cc new file mode 100644 index 000000000000..f032d388b724 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/test/test.cc @@ -0,0 +1,87 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include "graph_planner.h" + +#include + +std::string get_dir_name() +{ + // Get the directory of this source file + std::string current_dir = __FILE__; + size_t pos = current_dir.find_last_of("/"); + current_dir = current_dir.substr(0, pos); + return current_dir; +} + +void check_path_exits(const std::string &path) +{ + // split path by ':' + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + paths.push_back(sub_path); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + paths.push_back(sub_path); + + for (const auto &p : paths) + { + struct stat buffer; + if (stat(p.c_str(), &buffer) != 0) + { + std::cerr << "Path not exists: " << p << std::endl; + exit(1); + } + } + std::cout << "Path exists: " << path << std::endl; +} + +int main(int argc, char **argv) +{ + // Check if the correct number of arguments is provided + if (argc != 7) + { + std::cerr << "Usage: " << argv[0] + << " " << std::endl; + return 1; + } + + std::string java_class_path = argv[1]; + std::string jna_class_path = argv[2]; + std::string graph_schema_yaml = argv[3]; + std::string graph_statistic_json = argv[4]; + + // check director or file exists + check_path_exits(java_class_path); + check_path_exits(jna_class_path); + check_path_exits(graph_schema_yaml); + check_path_exits(graph_statistic_json); + + gs::GraphPlannerWrapper graph_planner_wrapper( + java_class_path, jna_class_path, graph_schema_yaml, graph_statistic_json); + + std::string cypher_query_string = argv[5]; + std::string config_path = argv[6]; + auto plan = + graph_planner_wrapper.CompilePlan(config_path, cypher_query_string); + std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; + std::cout << "schema: " << plan.result_schema << std::endl; + return 0; +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java index 5469e8ad55bd..e3800e80eba1 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java @@ -18,7 +18,8 @@ package com.alibaba.graphscope.common.ir; -import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.jna._native.GraphPlannerJNI; +import com.alibaba.graphscope.common.jna._native.JNIPlan; import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; import org.junit.Test; @@ -26,13 +27,13 @@ public class JNITest { @Test public void test() throws Exception { - Object[] objects = - GraphPlanner.generatePhysicalPlan( + JNIPlan objects = + GraphPlannerJNI.compilePlan( "conf/ir.compiler.properties", "Match (n) Return n, count(n)"); GraphAlgebraPhysical.PhysicalPlan plan = - GraphAlgebraPhysical.PhysicalPlan.parseFrom((byte[]) objects[0]); + GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); System.out.println(plan); - String resultSchema = (String) objects[1]; + String resultSchema = objects.resultSchemaYaml; System.out.println(resultSchema); } } diff --git a/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml b/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml new file mode 100644 index 000000000000..84ac110dffb3 --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml @@ -0,0 +1,46 @@ +log_level: INFO +verbose_level: 10 +default_graph: modern_graph +compute_engine: + type: hiactor + workers: + - localhost:10000 + thread_num_per_worker: 1 + store: + type: cpp-mcsr + metadata_store: + type: file # file/sqlite/etcd +compiler: + physical: + opt: + config: proto + planner: + is_on: true + opt: CBO + rules: + - FilterIntoJoinRule + - FilterMatchRule + - ExtendIntersectRule + - ExpandGetVFusionRule + meta: + reader: + schema: + uri: http://localhost:7777/v1/service/status + interval: 1000 # ms + statistics: + uri: http://localhost:7777/v1/graph/%s/statistics + interval: 86400000 # ms + endpoint: + default_listen_address: localhost + bolt_connector: + disabled: false + port: 7687 + gremlin_connector: + disabled: false + port: 8182 + query_timeout: 40000 + gremlin_script_language_name: antlr_gremlin_calcite +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 10000 diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index f79b9c0666ad..01519069f73b 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -54,6 +54,12 @@ false + + common + executor/engine/pegasus/clients/java/client + compiler + assembly + @@ -75,27 +81,11 @@ + false - - graphscope - - true - - - v6d - - - assembly - common - executor - frontend - executor/engine/pegasus/clients/java/client - compiler - - groot From 861779351b51e52939ad60872a0106e8fff6057b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 11 Dec 2024 21:20:52 +0800 Subject: [PATCH 14/27] refine jni interface, accept graph schema and statistics in string --- .../common/ir/tools/GraphPlanner.java | 36 ++++++----- .../common/jna/_native/GraphPlannerJNI.java | 64 ++++++++++++++++++- .../compiler/src/main/native/README.md | 30 +++++---- .../compiler/src/main/native/graph_planner.cc | 26 +++++--- .../compiler/src/main/native/graph_planner.h | 18 +++--- .../compiler/src/main/native/test/test.cc | 37 +++++++++-- .../alibaba/graphscope/common/ir/JNITest.java | 2 +- 7 files changed, 158 insertions(+), 55 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 77dc36d2a893..bc4ef8184a88 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -225,26 +225,32 @@ private static Configs createExtraConfigs(@Nullable String extraYamlFile) throws return new Configs(keyValueMap); } - private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) - throws IOException { - URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); - if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - } else if (schemaUri.getScheme().equals("http")) { - return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); - } - throw new IllegalArgumentException( - "unknown graph meta reader mode: " + schemaUri.getScheme()); + public interface IrMetaFetcherFactory { + IrMetaFetcher create(Configs configs, IrMetaTracker tracker) throws IOException; + + IrMetaFetcherFactory DEFAULT = + (configs, tracker) -> { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + }; } - public static Summary generatePlan(String config_path, String query_string) throws Exception { - Configs configs = Configs.Factory.create(config_path); + public static Summary generatePlan( + String configPath, String queryString, IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + Configs configs = Configs.Factory.create(configPath); GraphRelOptimizer optimizer = new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); - IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); + IrMetaFetcher metaFetcher = metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); GraphPlanner planner = new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); - PlannerInstance instance = planner.instance(query_string, metaFetcher.fetch().get()); + PlannerInstance instance = planner.instance(queryString, metaFetcher.fetch().get()); Summary summary = instance.plan(); return summary; } @@ -270,7 +276,7 @@ public static void main(String[] args) throws Exception { String query = builder.toString(); reader.close(); - Summary summary = generatePlan(args[0], query); + Summary summary = generatePlan(args[0], query, IrMetaFetcherFactory.DEFAULT); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; FileOutputStream fos = new FileOutputStream(args[2]); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java index 3764242c3edc..3a34facb2a24 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java @@ -19,26 +19,48 @@ package com.alibaba.graphscope.common.jna._native; import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.GraphId; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphStatistics; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaInputStream; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaSpec; import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; public class GraphPlannerJNI { /** * Provide a java-side implementation to compile the query in string to a physical plan - * @param config + * @param configPath * @param query + * * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, * resultSchemaYaml defines the result specification of the query in yaml format * @throws Exception */ - public static JNIPlan compilePlan(String config, String query) throws Exception { - GraphPlanner.Summary summary = GraphPlanner.generatePlan(config, query); + public static JNIPlan compilePlan( + String configPath, String query, String schemaYaml, String statsJson) throws Exception { + GraphPlanner.Summary summary = + GraphPlanner.generatePlan( + configPath, + query, + (Configs configs, IrMetaTracker tracker) -> + new StaticIrMetaFetcher( + new StringMetaReader(schemaYaml, statsJson), tracker)); LogicalPlan logicalPlan = summary.getLogicalPlan(); PhysicalPlan physicalPlan = summary.getPhysicalPlan(); StoredProcedureMeta procedureMeta = @@ -51,4 +73,40 @@ public static JNIPlan compilePlan(String config, String query) throws Exception StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); return new JNIPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); } + + static class StringMetaReader implements IrMetaReader { + private final String schemaYaml; + private final String statsJson; + + public StringMetaReader(String schemaYaml, String statsJson) { + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + } + + @Override + public IrMeta readMeta() throws IOException { + IrGraphSchema graphSchema = + new IrGraphSchema( + new SchemaInputStream( + new ByteArrayInputStream( + schemaYaml.getBytes(StandardCharsets.UTF_8)), + SchemaSpec.Type.FLEX_IN_YAML)); + return new IrMeta( + graphSchema, + new GraphStoredProcedures( + new ByteArrayInputStream(schemaYaml.getBytes(StandardCharsets.UTF_8)), + this)); + } + + @Override + public GraphStatistics readStats(GraphId graphId) throws IOException { + return new IrGraphStatistics( + new ByteArrayInputStream(statsJson.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public boolean syncStatsEnabled(GraphId graphId) throws IOException { + return false; + } + } } diff --git a/interactive_engine/compiler/src/main/native/README.md b/interactive_engine/compiler/src/main/native/README.md index ca814ae2d4bf..0d8d4b834912 100644 --- a/interactive_engine/compiler/src/main/native/README.md +++ b/interactive_engine/compiler/src/main/native/README.md @@ -13,7 +13,7 @@ mvn clean package -DskipTests -Pgraph-planner-jni ### Step 2: Locate and Extract the Package -After the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents of the tarball: +After the build completes, a tarball named `graph-planner-jni.tar.gz` will be available in the `assembly/target` directory. Extract the contents of the tarball: ```bash cd assembly/target @@ -23,7 +23,7 @@ cd graph-planner-jni ### Step 3: Run the Example Binary -To demonstrate the usage of the JNI interface, an example binary test_graph_planner is provided. Use the following command to execute it: +To demonstrate the usage of the JNI interface, an example binary `test_graph_planner` is provided. Use the following command to execute it: ```bash # bin/test_graph_planner @@ -62,24 +62,28 @@ Below is a brief explanation of the interface provided by the example: * @brief Constructs a new GraphPlannerWrapper object * @param java_path Java class path * @param jna_path JNA library path - * @param graph_schema_yaml Path to the graph schema file in YAML format - * @param graph_statistic_json Path to the graph statistics file in JSON format + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) */ GraphPlannerWrapper(const std::string &java_path, const std::string &jna_path, - const std::string &graph_schema_yaml, + const std::string &graph_schema_yaml = "", const std::string &graph_statistic_json = ""); ``` ## Method ```cpp -/** - * @brief Compiles a Cypher query into a physical plan via JNI invocation. - * @param compiler_config_path Path to the compiler configuration file. - * @param cypher_query_string Cypher query string to compile. - * @return Physical plan in bytes and result schema in YAML format. - */ -Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, - const std::string &cypher_query_string); +/** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format + * @return The physical plan in bytes and result schema in yaml. + */ +Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) ``` diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc index 69da83c9c8ef..d5dbbba57973 100644 --- a/interactive_engine/compiler/src/main/native/graph_planner.cc +++ b/interactive_engine/compiler/src/main/native/graph_planner.cc @@ -239,18 +239,20 @@ namespace gs std::cout << "Expanded java path: " << expanded_java_path << std::endl; std::string jvm_options = "-Djava.class.path=" + expanded_java_path; jvm_options += " -Djna.library.path=" + jna_path; - jvm_options += " -Dgraph.schema=" + graph_schema_yaml; - if (!graph_statistic_json.empty()) - { - jvm_options += " -Dgraph.statistics=" + graph_statistic_json; - } + // jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + // if (!graph_statistic_json.empty()) + // { + // jvm_options += " -Dgraph.statistics=" + graph_statistic_json; + // } return jvm_options; } Plan compilePlanJNI(jclass graph_planner_clz_, jmethodID graph_planner_method_id_, JNIEnv *env, const std::string &compiler_config_path, - const std::string &cypher_query_string) + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) { jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), nullptr); @@ -262,10 +264,12 @@ namespace gs } jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + jstring param3 = env->NewStringUTF(graph_schema_yaml.c_str()); + jstring param4 = env->NewStringUTF(graph_statistic_json.c_str()); // invoke jvm static function to get results as Object[] jobject jni_plan = (jobject)env->CallStaticObjectMethod( - graph_planner_clz_, graph_planner_method_id_, param1, param2); + graph_planner_clz_, graph_planner_method_id_, param1, param2, param3, param4); if (env->ExceptionCheck()) { @@ -437,15 +441,19 @@ namespace gs * @brief Compile a cypher query to a physical plan by JNI invocation. * @param compiler_config_path The path of compiler config file. * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format * @return The physical plan in bytes and result schema in yaml. */ Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, - const std::string &cypher_query_string) + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) { #if (GRAPH_PLANNER_JNI_INVOKER) return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, jni_wrapper_.env(), compiler_config_path, - cypher_query_string); + cypher_query_string, graph_schema_yaml, graph_statistic_json); #else return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, graph_statistic_json_, compiler_config_path, diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h index 28ce4f15b0cd..613e9bc9abd5 100644 --- a/interactive_engine/compiler/src/main/native/graph_planner.h +++ b/interactive_engine/compiler/src/main/native/graph_planner.h @@ -63,18 +63,18 @@ namespace gs "com.alibaba.graphscope.common.jna._native.GraphPlannerJNI"; static constexpr const char *kGraphPlannerMethod = "compilePlan"; static constexpr const char *kGraphPlannerMethodSignature = - "(Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/common/jna/_native/JNIPlan;"; + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/common/jna/_native/JNIPlan;"; /** - * @brief Construct a new Graph Planner Wrapper object - * @param java_path java class path - * @param jna_path jna library path - * @param graph_schema_yaml graph schema path - * @param graph_statistic_json graph statistic path + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) */ GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, - const std::string &graph_schema_yaml, + const std::string &graph_schema_yaml = "", const std::string &graph_statistic_json = "") #if (GRAPH_PLANNER_JNI_INVOKER) : jni_wrapper_(generate_jvm_options( @@ -133,7 +133,9 @@ namespace gs * @return physical plan in string. */ Plan CompilePlan(const std::string &compiler_config_path, - const std::string &cypher_query_string); + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); private: std::string generate_jvm_options(const std::string java_path, diff --git a/interactive_engine/compiler/src/main/native/test/test.cc b/interactive_engine/compiler/src/main/native/test/test.cc index f032d388b724..f0e161f4a053 100644 --- a/interactive_engine/compiler/src/main/native/test/test.cc +++ b/interactive_engine/compiler/src/main/native/test/test.cc @@ -14,6 +14,9 @@ limitations under the License. */ #include "graph_planner.h" +#include +#include + #include std::string get_dir_name() @@ -53,6 +56,25 @@ void check_path_exits(const std::string &path) std::cout << "Path exists: " << path << std::endl; } +std::string read_string_from_file(const std::string &file_path) +{ + std::ifstream inputFile(file_path); // Open the file for reading + + if (!inputFile.is_open()) + { + std::cerr << "Error: Could not open the file " << file_path << std::endl; + exit(1); + } + // Use a stringstream to read the entire content of the file + std::ostringstream buffer; + buffer << inputFile.rdbuf(); // Read the file stream into the stringstream + + std::string fileContent = buffer.str(); // Get the string from the stringstream + inputFile.close(); // Close the file + + return fileContent; +} + int main(int argc, char **argv) { // Check if the correct number of arguments is provided @@ -65,22 +87,25 @@ int main(int argc, char **argv) std::string java_class_path = argv[1]; std::string jna_class_path = argv[2]; - std::string graph_schema_yaml = argv[3]; - std::string graph_statistic_json = argv[4]; + std::string graph_schema_path = argv[3]; + std::string graph_statistic_path = argv[4]; // check director or file exists check_path_exits(java_class_path); check_path_exits(jna_class_path); - check_path_exits(graph_schema_yaml); - check_path_exits(graph_statistic_json); + check_path_exits(graph_schema_path); + check_path_exits(graph_statistic_path); gs::GraphPlannerWrapper graph_planner_wrapper( - java_class_path, jna_class_path, graph_schema_yaml, graph_statistic_json); + java_class_path, jna_class_path); + + std::string schema_content = read_string_from_file(graph_schema_path); + std::string statistic_content = read_string_from_file(graph_statistic_path); std::string cypher_query_string = argv[5]; std::string config_path = argv[6]; auto plan = - graph_planner_wrapper.CompilePlan(config_path, cypher_query_string); + graph_planner_wrapper.CompilePlan(config_path, cypher_query_string, schema_content, statistic_content); std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; std::cout << "schema: " << plan.result_schema << std::endl; return 0; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java index e3800e80eba1..dc0b618d8f79 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java @@ -29,7 +29,7 @@ public class JNITest { public void test() throws Exception { JNIPlan objects = GraphPlannerJNI.compilePlan( - "conf/ir.compiler.properties", "Match (n) Return n, count(n)"); + "conf/ir.compiler.properties", "Match (n) Return n, count(n)", "", ""); GraphAlgebraPhysical.PhysicalPlan plan = GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); System.out.println(plan); From e217d75ad21984a9964f178d23f1ddcffa6ec22f Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 12 Dec 2024 18:28:08 +0800 Subject: [PATCH 15/27] expose 'compilePlan' interface by restful api --- .../compiler/conf/application.yaml | 22 + interactive_engine/compiler/diff | 1199 ----------------- interactive_engine/compiler/pom.xml | 4 + .../JNIPlan.java => sdk/GraphPlan.java} | 12 +- .../PlanUtils.java} | 8 +- .../sdk/examples/TestGraphPlanner.java | 91 ++ .../sdk/restful/GraphPlanRequest.java | 69 + .../sdk/restful/GraphPlanResponse.java | 57 + .../sdk/restful/GraphPlannerController.java | 54 + .../sdk/restful/GraphPlannerService.java | 29 + .../compiler/src/main/native/README.md | 132 ++ .../compiler/src/main/native/graph_planner.h | 6 +- .../alibaba/graphscope/common/ir/JNITest.java | 29 +- .../graphscope/sdk/GraphPlannerTest.java | 86 ++ interactive_engine/pom.xml | 9 + 15 files changed, 579 insertions(+), 1228 deletions(-) create mode 100644 interactive_engine/compiler/conf/application.yaml delete mode 100644 interactive_engine/compiler/diff rename interactive_engine/compiler/src/main/java/com/alibaba/graphscope/{common/jna/_native/JNIPlan.java => sdk/GraphPlan.java} (77%) rename interactive_engine/compiler/src/main/java/com/alibaba/graphscope/{common/jna/_native/GraphPlannerJNI.java => sdk/PlanUtils.java} (95%) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java diff --git a/interactive_engine/compiler/conf/application.yaml b/interactive_engine/compiler/conf/application.yaml new file mode 100644 index 000000000000..d5e9d11a3746 --- /dev/null +++ b/interactive_engine/compiler/conf/application.yaml @@ -0,0 +1,22 @@ +# +# /* +# * Copyright 2020 Alibaba Group Holding Limited. +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +server: + port: 8080 +app: + name: GraphPlannerService \ No newline at end of file diff --git a/interactive_engine/compiler/diff b/interactive_engine/compiler/diff deleted file mode 100644 index 30e8edec8263..000000000000 --- a/interactive_engine/compiler/diff +++ /dev/null @@ -1,1199 +0,0 @@ -diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml -index 0e4495ca64..e8efca65d7 100644 ---- a/interactive_engine/assembly/graph-planner-jni.xml -+++ b/interactive_engine/assembly/graph-planner-jni.xml -@@ -16,6 +16,16 @@ - - - ${project.parent.basedir}/target/native/ -+ -+ libgraph_planner.* -+ -+ native -+ -+ -+ ${project.parent.basedir}/target/native/ -+ -+ test_graph_planner -+ - bin - 0755 - -@@ -23,6 +33,13 @@ - ${project.parent.basedir}/compiler/target/libs/ - libs - -+ -+ ${project.parent.basedir}/compiler/target/ -+ -+ compiler-0.0.1-SNAPSHOT.jar -+ -+ libs -+ - - ${project.parent.basedir}/compiler/conf - conf -diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties -index 992f828a2b..5a13dc9d4f 100644 ---- a/interactive_engine/compiler/conf/ir.compiler.properties -+++ b/interactive_engine/compiler/conf/ir.compiler.properties -@@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json - graph.store: exp - - graph.planner.is.on: true --graph.planner.opt: RBO -+graph.planner.opt: CBO - graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule - - # set statistics access uri --# graph.statistics: src/test/resources/statistics/modern_statistics.json -+graph.statistics: src/test/resources/statistics/modern_statistics.json - - # set stored procedures directory path - # graph.stored.procedures: -@@ -60,7 +60,7 @@ calcite.default.charset: UTF-8 - # gremlin.script.language.name: antlr_gremlin_traversal - - # the output plan format, can be ffi(default) or proto --# graph.physical.opt: ffi -+graph.physical.opt: proto - - # set the max capacity of the result streaming buffer for each query - # per.query.stream.buffer.max.capacity: 256 -diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java -index 991ae7b89d..77dc36d2a8 100644 ---- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java -+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java -@@ -249,23 +249,6 @@ public class GraphPlanner { - return summary; - } - -- public static Object[] generatePhysicalPlan(String config_path, String query_string) -- throws Exception { -- Summary summary = generatePlan(config_path, query_string); -- LogicalPlan logicalPlan = summary.getLogicalPlan(); -- PhysicalPlan physicalPlan = summary.physicalPlan; -- Configs extraConfigs = createExtraConfigs(null); -- StoredProcedureMeta procedureMeta = -- new StoredProcedureMeta( -- extraConfigs, -- query_string, -- logicalPlan.getOutputType(), -- logicalPlan.getDynamicParams()); -- ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); -- StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); -- return new Object[] {physicalPlan.getContent(), new String(metaStream.toByteArray())}; -- } -- - public static void main(String[] args) throws Exception { - if (args.length < 4 - || args[0].isEmpty() -diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java -index bea8926ed7..ee1766e855 100644 ---- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java -+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java -@@ -1,2 +1,55 @@ --package com.alibaba.graphscope.common.jna._native;public class GraphPlannerJNI { -+/* -+ * -+ * * Copyright 2020 Alibaba Group Holding Limited. -+ * * -+ * * Licensed under the Apache License, Version 2.0 (the "License"); -+ * * you may not use this file except in compliance with the License. -+ * * You may obtain a copy of the License at -+ * * -+ * * http://www.apache.org/licenses/LICENSE-2.0 -+ * * -+ * * Unless required by applicable law or agreed to in writing, software -+ * * distributed under the License is distributed on an "AS IS" BASIS, -+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * * See the License for the specific language governing permissions and -+ * * limitations under the License. -+ * -+ */ -+ -+package com.alibaba.graphscope.common.jna._native; -+ -+import com.alibaba.graphscope.common.config.Configs; -+import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; -+import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; -+import com.alibaba.graphscope.common.ir.tools.GraphPlanner; -+import com.alibaba.graphscope.common.ir.tools.LogicalPlan; -+import com.google.common.collect.ImmutableMap; -+ -+import java.io.ByteArrayOutputStream; -+ -+public class GraphPlannerJNI { -+ /** -+ * Provide a java-side implementation to compile the query in string to a physical plan -+ * @param config -+ * @param query -+ * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, -+ * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, -+ * resultSchemaYaml defines the result specification of the query in yaml format -+ * @throws Exception -+ */ -+ public static JNIPlan compilePlan(String config, String query) -+ throws Exception { -+ GraphPlanner.Summary summary = GraphPlanner.generatePlan(config, query); -+ LogicalPlan logicalPlan = summary.getLogicalPlan(); -+ PhysicalPlan physicalPlan = summary.getPhysicalPlan(); -+ StoredProcedureMeta procedureMeta = -+ new StoredProcedureMeta( -+ new Configs(ImmutableMap.of()), -+ query, -+ logicalPlan.getOutputType(), -+ logicalPlan.getDynamicParams()); -+ ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); -+ StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); -+ return new JNIPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); -+ } - } -diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java -index 93d4e3615a..689c9c74ef 100644 ---- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java -+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java -@@ -1,2 +1,37 @@ --package com.alibaba.graphscope.common.jna._native;public class JNIPlan { -+/* -+ * -+ * * Copyright 2020 Alibaba Group Holding Limited. -+ * * -+ * * Licensed under the Apache License, Version 2.0 (the "License"); -+ * * you may not use this file except in compliance with the License. -+ * * You may obtain a copy of the License at -+ * * -+ * * http://www.apache.org/licenses/LICENSE-2.0 -+ * * -+ * * Unless required by applicable law or agreed to in writing, software -+ * * distributed under the License is distributed on an "AS IS" BASIS, -+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * * See the License for the specific language governing permissions and -+ * * limitations under the License. -+ * -+ */ -+ -+package com.alibaba.graphscope.common.jna._native; -+ -+public class JNIPlan { -+ public final byte[] physicalBytes; -+ public final String resultSchemaYaml; -+ -+ public JNIPlan(byte[] physicalBytes, String resultSchemaYaml) { -+ this.physicalBytes = physicalBytes; -+ this.resultSchemaYaml = resultSchemaYaml; -+ } -+ -+ public byte[] getPhysicalBytes() { -+ return physicalBytes; -+ } -+ -+ public String getResultSchemaYaml() { -+ return resultSchemaYaml; -+ } - } -diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc -index 417169a130..0b4a304fd6 100644 ---- a/interactive_engine/compiler/src/main/native/graph_planner.cc -+++ b/interactive_engine/compiler/src/main/native/graph_planner.cc -@@ -22,365 +22,429 @@ limitations under the License. - #include "graph_planner.h" - #include - --namespace gs { -+namespace gs -+{ - #if (GRAPH_PLANNER_JNI_INVOKER) --namespace jni { -- --static JavaVM *_jvm = NULL; -- --JavaVM *CreateJavaVM(const std::string &jvm_options) { -- const char *p, *q; -- const char *jvm_opts; -- if (jvm_options.empty()) { -- jvm_opts = getenv("FLEX_JVM_OPTS"); -- } else { -- jvm_opts = jvm_options.c_str(); -- } -- if (jvm_opts == NULL) { -- std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; -- return NULL; -- } -- std::cout << "Jvm opts str: " << jvm_opts << std::endl; -+ namespace jni -+ { -+ -+ static JavaVM *_jvm = NULL; -+ -+ JavaVM *CreateJavaVM(const std::string &jvm_options) -+ { -+ const char *p, *q; -+ const char *jvm_opts; -+ if (jvm_options.empty()) -+ { -+ jvm_opts = getenv("FLEX_JVM_OPTS"); -+ } -+ else -+ { -+ jvm_opts = jvm_options.c_str(); -+ } -+ if (jvm_opts == NULL) -+ { -+ std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; -+ return NULL; -+ } -+ std::cout << "Jvm opts str: " << jvm_opts << std::endl; - -- if (*jvm_opts == '\0') -- return NULL; -+ if (*jvm_opts == '\0') -+ return NULL; - -- int num_of_opts = 1; -- for (const char *p = jvm_opts; *p; p++) { -- if (*p == ' ') -- num_of_opts++; -- } -+ int num_of_opts = 1; -+ for (const char *p = jvm_opts; *p; p++) -+ { -+ if (*p == ' ') -+ num_of_opts++; -+ } - -- if (num_of_opts == 0) -- return NULL; -+ if (num_of_opts == 0) -+ return NULL; -+ -+ JavaVM *jvm = NULL; -+ JNIEnv *env = NULL; -+ int i = 0; -+ int status = 1; -+ JavaVMInitArgs vm_args; -+ -+ JavaVMOption *options = new JavaVMOption[num_of_opts]; -+ memset(options, 0, sizeof(JavaVMOption) * num_of_opts); -+ -+ for (p = q = jvm_opts;; p++) -+ { -+ if (*p == ' ' || *p == '\0') -+ { -+ if (q >= p) -+ { -+ goto ret; -+ } -+ char *opt = new char[p - q + 1]; -+ memcpy(opt, q, p - q); -+ opt[p - q] = '\0'; -+ options[i++].optionString = opt; -+ q = p + 1; // assume opts are separated by single space -+ if (*p == '\0') -+ break; -+ } -+ } - -- JavaVM *jvm = NULL; -- JNIEnv *env = NULL; -- int i = 0; -- int status = 1; -- JavaVMInitArgs vm_args; -+ memset(&vm_args, 0, sizeof(vm_args)); -+ vm_args.version = JNI_VERSION_1_8; -+ vm_args.nOptions = num_of_opts; -+ vm_args.options = options; - -- JavaVMOption *options = new JavaVMOption[num_of_opts]; -- memset(options, 0, sizeof(JavaVMOption) * num_of_opts); -+ status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); -+ if (status == JNI_OK) -+ { -+ std::cout << "Create java virtual machine successfully." << std::endl; -+ } -+ else if (status == JNI_EEXIST) -+ { -+ std::cout << "JNI evn already exists." << std::endl; -+ } -+ else -+ { -+ std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" -+ << status << ")" << std::endl; -+ } - -- for (p = q = jvm_opts;; p++) { -- if (*p == ' ' || *p == '\0') { -- if (q >= p) { -- goto ret; -+ ret: -+ for (int i = 0; i < num_of_opts; i++) -+ { -+ delete[] options[i].optionString; - } -- char *opt = new char[p - q + 1]; -- memcpy(opt, q, p - q); -- opt[p - q] = '\0'; -- options[i++].optionString = opt; -- q = p + 1; // assume opts are separated by single space -- if (*p == '\0') -- break; -+ delete[] options; -+ return jvm; - } -- } -- -- memset(&vm_args, 0, sizeof(vm_args)); -- vm_args.version = JNI_VERSION_1_8; -- vm_args.nOptions = num_of_opts; -- vm_args.options = options; -- -- status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); -- if (status == JNI_OK) { -- std::cout << "Create java virtual machine successfully." << std::endl; -- } else if (status == JNI_EEXIST) { -- std::cout << "JNI evn already exists." << std::endl; -- } else { -- std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" -- << status << ")" << std::endl; -- } - --ret: -- for (int i = 0; i < num_of_opts; i++) { -- delete[] options[i].optionString; -- } -- delete[] options; -- return jvm; --} -- --// One process can only create jvm for once. --JavaVM *GetJavaVM(const std::string jvm_options = "") { -- if (_jvm == NULL) { -- // Try to find whether there exists one javaVM -- jsize nVMs; -- JNI_GetCreatedJavaVMs(NULL, 0, -- &nVMs); // 1. just get the required array length -- std::cout << "Found " << nVMs << " VMs existing in this process." -- << std::endl; -- JavaVM **buffer = new JavaVM *[nVMs]; -- JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data -- for (auto i = 0; i < nVMs; ++i) { -- if (buffer[i] != NULL) { -- _jvm = buffer[i]; -- std::cout << "Found index " << i << " VM non null " -- << reinterpret_cast(_jvm) << std::endl; -- return _jvm; -+ // One process can only create jvm for once. -+ JavaVM *GetJavaVM(const std::string jvm_options = "") -+ { -+ if (_jvm == NULL) -+ { -+ // Try to find whether there exists one javaVM -+ jsize nVMs; -+ JNI_GetCreatedJavaVMs(NULL, 0, -+ &nVMs); // 1. just get the required array length -+ std::cout << "Found " << nVMs << " VMs existing in this process." -+ << std::endl; -+ JavaVM **buffer = new JavaVM *[nVMs]; -+ JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data -+ for (auto i = 0; i < nVMs; ++i) -+ { -+ if (buffer[i] != NULL) -+ { -+ _jvm = buffer[i]; -+ std::cout << "Found index " << i << " VM non null " -+ << reinterpret_cast(_jvm) << std::endl; -+ return _jvm; -+ } -+ } -+ _jvm = CreateJavaVM(jvm_options); -+ std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; - } -+ return _jvm; - } -- _jvm = CreateJavaVM(jvm_options); -- std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; -- } -- return _jvm; --} - --JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} -+ JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} - --JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) { -- if (!GetJavaVM(jvm_options)) { -- return; -- } -- int status = -- GetJavaVM(jvm_options) -- ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); -- if (status != JNI_OK) { -- std::cerr << "Error attach current thread: " << status << std::endl; -- } --} -+ JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) -+ { -+ if (!GetJavaVM(jvm_options)) -+ { -+ return; -+ } -+ int status = -+ GetJavaVM(jvm_options) -+ ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); -+ if (status != JNI_OK) -+ { -+ std::cerr << "Error attach current thread: " << status << std::endl; -+ } -+ } - --JNIEnvMark::~JNIEnvMark() { -- if (_env) { -- GetJavaVM()->DetachCurrentThread(); -- } --} -+ JNIEnvMark::~JNIEnvMark() -+ { -+ if (_env) -+ { -+ GetJavaVM()->DetachCurrentThread(); -+ } -+ } - --JNIEnv *JNIEnvMark::env() { return _env; } -+ JNIEnv *JNIEnvMark::env() { return _env; } - --} // namespace jni -+ } // namespace jni - - #endif - --std::vector list_files(const std::string &path) { -- // list all files in the directory -- std::vector files; -- for (const auto &entry : std::filesystem::directory_iterator(path)) { -- files.push_back(entry.path().string()); -+ std::vector list_files(const std::string &path) -+ { -+ // list all files in the directory -+ std::vector files; -+ for (const auto &entry : std::filesystem::directory_iterator(path)) -+ { -+ files.push_back(entry.path().string()); -+ } -+ return files; - } -- return files; --} - --void iterate_over_director(const std::string& dir_or_path, std::vector& output_paths){ -- if (dir_or_path.empty()) { -- return; -+ void iterate_over_director(const std::string &dir_or_path, std::vector &output_paths) -+ { -+ if (dir_or_path.empty()) -+ { -+ return; -+ } -+ if (std::filesystem::is_directory(dir_or_path)) -+ { -+ auto files = list_files(dir_or_path); -+ output_paths.insert(output_paths.end(), files.begin(), files.end()); -+ } -+ else -+ { -+ output_paths.push_back(dir_or_path); -+ } - } -- if (std::filesystem::is_directory(dir_or_path)) { -- auto files = list_files(dir_or_path); -- output_paths.insert(output_paths.end(), files.begin(), files.end()); -- } else { -- output_paths.push_back(dir_or_path); -- } --} -- --std::string GraphPlannerWrapper::expand_directory(const std::string &path) { -- std::vector paths; -- std::string::size_type start = 0; -- std::string::size_type end = path.find(':'); -- while (end != std::string::npos) { -- auto sub_path = path.substr(start, end - start); -+ -+ std::string GraphPlannerWrapper::expand_directory(const std::string &path) -+ { -+ std::vector paths; -+ std::string::size_type start = 0; -+ std::string::size_type end = path.find(':'); -+ while (end != std::string::npos) -+ { -+ auto sub_path = path.substr(start, end - start); -+ iterate_over_director(sub_path, paths); -+ start = end + 1; -+ end = path.find(':', start); -+ } -+ auto sub_path = path.substr(start); - iterate_over_director(sub_path, paths); -- start = end + 1; -- end = path.find(':', start); -- } -- auto sub_path = path.substr(start); -- iterate_over_director(sub_path, paths); -- std::stringstream ss; -- for (const auto &p : paths) { -- ss << p << ":"; -+ std::stringstream ss; -+ for (const auto &p : paths) -+ { -+ ss << p << ":"; -+ } -+ return ss.str(); - } -- return ss.str(); --} - - #if (GRAPH_PLANNER_JNI_INVOKER) - --std::string GraphPlannerWrapper::generate_jvm_options( -- const std::string java_path, const std::string &jna_path, -- const std::string &graph_schema_yaml, -- const std::string &graph_statistic_json) { -- auto expanded_java_path = expand_directory(java_path); -- std::cout << "Expanded java path: " << expanded_java_path << std::endl; -- std::string jvm_options = "-Djava.class.path=" + expanded_java_path; -- jvm_options += " -Djna.library.path=" + jna_path; -- jvm_options += " -Dgraph.schema=" + graph_schema_yaml; -- if (!graph_statistic_json.empty()) { -- jvm_options += " -Dgraph.statistic=" + graph_statistic_json; -- } -- return jvm_options; --} -- --Plan compilePlanJNI(jclass graph_planner_clz_, -- jmethodID graph_planner_method_id_, JNIEnv *env, -- const std::string &compiler_config_path, -- const std::string &cypher_query_string) { -- jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), -- nullptr); -- Plan plan; -- if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) { -- std::cerr << "Invalid GraphPlannerWrapper." << std::endl; -- return plan; -+ std::string GraphPlannerWrapper::generate_jvm_options( -+ const std::string java_path, const std::string &jna_path, -+ const std::string &graph_schema_yaml, -+ const std::string &graph_statistic_json) -+ { -+ auto expanded_java_path = expand_directory(java_path); -+ std::cout << "Expanded java path: " << expanded_java_path << std::endl; -+ std::string jvm_options = "-Djava.class.path=" + expanded_java_path; -+ jvm_options += " -Djna.library.path=" + jna_path; -+ jvm_options += " -Dgraph.schema=" + graph_schema_yaml; -+ if (!graph_statistic_json.empty()) -+ { -+ jvm_options += " -Dgraph.statistic=" + graph_statistic_json; -+ } -+ return jvm_options; - } -- jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); -- jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); - -- // invoke jvm static function to get results as Object[] -- jobjectArray resultArray = (jobjectArray)env->CallStaticObjectMethod( -- graph_planner_clz_, graph_planner_method_id_, param1, param2); -+ Plan compilePlanJNI(jclass graph_planner_clz_, -+ jmethodID graph_planner_method_id_, JNIEnv *env, -+ const std::string &compiler_config_path, -+ const std::string &cypher_query_string) -+ { -+ jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), -+ nullptr); -+ Plan plan; -+ if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) -+ { -+ std::cerr << "Invalid GraphPlannerWrapper." << std::endl; -+ return plan; -+ } -+ jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); -+ jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); -+ -+ // invoke jvm static function to get results as Object[] -+ jobject jni_plan = (jobject)env->CallStaticObjectMethod( -+ graph_planner_clz_, graph_planner_method_id_, param1, param2); -+ -+ if (env->ExceptionCheck()) -+ { -+ env->ExceptionDescribe(); -+ env->ExceptionClear(); -+ std::cerr << "Error in calling GraphPlanner." << std::endl; -+ return plan; -+ } - -- if (env->ExceptionCheck()) { -- env->ExceptionDescribe(); -- env->ExceptionClear(); -- std::cerr << "Error in calling GraphPlanner." << std::endl; -- return plan; -- } -+ jmethodID method1 = env->GetMethodID( -+ env->GetObjectClass(jni_plan), "getPhysicalBytes", "()[B"); -+ jmethodID method2 = env->GetMethodID( -+ env->GetObjectClass(jni_plan), "getResultSchemaYaml", "()Ljava/lang/String;"); -+ -+ // 0-th object is the physical plan in byte array -+ jbyteArray res1 = (jbyteArray)env->CallObjectMethod(jni_plan, method1); -+ // 1-th object is the result schema in yaml format -+ jstring res2 = (jstring)env->CallObjectMethod(jni_plan, method2); -+ -+ if (res1 == NULL || res2 == NULL) -+ { -+ std::cerr << "Fail to generate plan." << std::endl; -+ return plan; -+ } -+ jbyte *str = env->GetByteArrayElements(res1, NULL); -+ jsize len = env->GetArrayLength(res1); -+ std::cout << "Physical plan size: " << len; - -- // 0-th object is the physical plan in byte array -- jbyteArray res = (jbyteArray)env->GetObjectArrayElement(resultArray, 0); -- // 1-th object is the result schema in yaml format -- jstring res2 = (jstring)env->GetObjectArrayElement(resultArray, 1); -+ plan.physical_plan.ParseFromArray(str, len); -+ plan.result_schema = env->GetStringUTFChars(res2, NULL); -+ -+ env->ReleaseByteArrayElements(res1, str, 0); -+ env->DeleteLocalRef(param1); -+ env->DeleteLocalRef(param2); -+ env->DeleteLocalRef(res1); -+ // remove new added jni objects -+ env->DeleteLocalRef(res2); -+ env->DeleteLocalRef(jni_plan); - -- if (res == NULL || res2 == NULL) { -- std::cerr << "Fail to generate plan." << std::endl; - return plan; - } -- jbyte *str = env->GetByteArrayElements(res, NULL); -- jsize len = env->GetArrayLength(res); -- std::cout << "Physical plan size: " << len; -- -- plan.physical_plan.ParseFromArray(str, len); -- plan.result_schema = env->GetStringUTFChars(res2, NULL); -- -- env->ReleaseByteArrayElements(res, str, 0); -- env->DeleteLocalRef(param1); -- env->DeleteLocalRef(param2); -- env->DeleteLocalRef(res); -- // remove new added jni objects -- env->DeleteLocalRef(res2); -- env->DeleteLocalRef(resultArray); -- -- return plan; --} - #endif - - #if (!GRAPH_PLANNER_JNI_INVOKER) - --void write_query_to_pipe(const std::string &path, -- const std::string &query_str) { -- std::cout << "write_query_to_pipe: " << path << std::endl; -- -- // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -- int fd_to_java = open(path.c_str(), O_WRONLY); -- if (fd_to_java < 0) { -- std::cerr << "Fail to open pipe: " << path << std::endl; -- return; -+ void write_query_to_pipe(const std::string &path, -+ const std::string &query_str) -+ { -+ std::cout << "write_query_to_pipe: " << path << std::endl; -+ -+ // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -+ int fd_to_java = open(path.c_str(), O_WRONLY); -+ if (fd_to_java < 0) -+ { -+ std::cerr << "Fail to open pipe: " << path << std::endl; -+ return; -+ } -+ std::cout << "open pipe done" << std::endl; -+ auto len = write(fd_to_java, query_str.c_str(), query_str.size()); -+ if (len != (int)query_str.size()) -+ { -+ std::cerr << "Fail to write query to pipe:" << len << std::endl; -+ return; -+ } -+ std::cout << "write_query_to_pipe done: " << len << std::endl; -+ close(fd_to_java); - } -- std::cout << "open pipe done" << std::endl; -- auto len = write(fd_to_java, query_str.c_str(), query_str.size()); -- if (len != (int)query_str.size()) { -- std::cerr << "Fail to write query to pipe:" << len << std::endl; -- return; -+ -+ void write_query_to_file(const std::string &path, -+ const std::string &query_str) -+ { -+ std::ofstream query_file(path); -+ query_file << query_str; -+ query_file.close(); - } -- std::cout << "write_query_to_pipe done: " << len << std::endl; -- close(fd_to_java); --} -- --void write_query_to_file(const std::string &path, -- const std::string &query_str) { -- std::ofstream query_file(path); -- query_file << query_str; -- query_file.close(); --} -- --physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) { -- std::cout << "plan str size: " << plan_str.size() << std::endl; -- physical::PhysicalPlan plan; -- if (!plan.ParseFromString(plan_str)) { -- std::cerr << "Fail to parse physical plan." << std::endl; -- return physical::PhysicalPlan(); -+ -+ physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) -+ { -+ std::cout << "plan str size: " << plan_str.size() << std::endl; -+ physical::PhysicalPlan plan; -+ if (!plan.ParseFromString(plan_str)) -+ { -+ std::cerr << "Fail to parse physical plan." << std::endl; -+ return physical::PhysicalPlan(); -+ } -+ return plan; - } -- return plan; --} -- --physical::PhysicalPlan --compilePlanSubprocess(const std::string &class_path, -- const std::string &jna_path, -- const std::string &graph_schema_yaml, -- const std::string &graph_statistic_json, -- const std::string &compiler_config_path, -- const std::string &cypher_query_string) { -- physical::PhysicalPlan physical_plan; -- auto random_prefix = std::to_string( -- std::chrono::system_clock::now().time_since_epoch().count()); -- std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; -- std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; -- std::cout << "dst_query_path: " << dst_query_path -- << " dst_output_file: " << dst_output_file << std::endl; -- mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -- mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -- -- pid_t pid = fork(); -- -- if (pid == 0) { -- const char *const command_string_array[] = {"java", -- "-cp", -- class_path.c_str(), -- jna_path.c_str(), -- graph_schema_yaml.c_str(), -- graph_statistic_json.c_str(), -- GRAPH_PLANNER_FULL_NAME, -- compiler_config_path.c_str(), -- dst_query_path.c_str(), -- dst_output_file.c_str(), -- "/tmp/temp.cypher.yaml", -- NULL}; -- execvp(command_string_array[0], -- const_cast(command_string_array)); -- } else if (pid < 0) { -- std::cerr << "Error in fork." << std::endl; -- } else { -- write_query_to_pipe(dst_query_path, cypher_query_string); -- -- int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); -- if (fd_from_java < 0) { -- std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; -- return physical_plan; -+ -+ physical::PhysicalPlan -+ compilePlanSubprocess(const std::string &class_path, -+ const std::string &jna_path, -+ const std::string &graph_schema_yaml, -+ const std::string &graph_statistic_json, -+ const std::string &compiler_config_path, -+ const std::string &cypher_query_string) -+ { -+ physical::PhysicalPlan physical_plan; -+ auto random_prefix = std::to_string( -+ std::chrono::system_clock::now().time_since_epoch().count()); -+ std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; -+ std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; -+ std::cout << "dst_query_path: " << dst_query_path -+ << " dst_output_file: " << dst_output_file << std::endl; -+ mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -+ mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); -+ -+ pid_t pid = fork(); -+ -+ if (pid == 0) -+ { -+ const char *const command_string_array[] = {"java", -+ "-cp", -+ class_path.c_str(), -+ jna_path.c_str(), -+ graph_schema_yaml.c_str(), -+ graph_statistic_json.c_str(), -+ GRAPH_PLANNER_FULL_NAME, -+ compiler_config_path.c_str(), -+ dst_query_path.c_str(), -+ dst_output_file.c_str(), -+ "/tmp/temp.cypher.yaml", -+ NULL}; -+ execvp(command_string_array[0], -+ const_cast(command_string_array)); - } -- std::vector stored_buffer; -- char buffer[128]; -- while (true) { -- ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); -- if (bytesRead <= 0) { -- break; -- } -- stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); -+ else if (pid < 0) -+ { -+ std::cerr << "Error in fork." << std::endl; - } -- physical_plan = readPhysicalPlan( -- std::string(stored_buffer.begin(), stored_buffer.end())); -- close(fd_from_java); -- -- int status; -- waitpid(pid, &status, 0); -- if (status != 0) { -- std::cerr << "Error in running command." << std::endl; -+ else -+ { -+ write_query_to_pipe(dst_query_path, cypher_query_string); -+ -+ int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); -+ if (fd_from_java < 0) -+ { -+ std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; -+ return physical_plan; -+ } -+ std::vector stored_buffer; -+ char buffer[128]; -+ while (true) -+ { -+ ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); -+ if (bytesRead <= 0) -+ { -+ break; -+ } -+ stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); -+ } -+ physical_plan = readPhysicalPlan( -+ std::string(stored_buffer.begin(), stored_buffer.end())); -+ close(fd_from_java); -+ -+ int status; -+ waitpid(pid, &status, 0); -+ if (status != 0) -+ { -+ std::cerr << "Error in running command." << std::endl; -+ } - } -+ unlink(dst_query_path.c_str()); -+ unlink(dst_output_file.c_str()); -+ return physical_plan; - } -- unlink(dst_query_path.c_str()); -- unlink(dst_output_file.c_str()); -- return physical_plan; --} - #endif - --Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, -- const std::string &cypher_query_string) { -+ Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, -+ const std::string &cypher_query_string) -+ { - #if (GRAPH_PLANNER_JNI_INVOKER) -- return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, -- jni_wrapper_.env(), compiler_config_path, -- cypher_query_string); -+ return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, -+ jni_wrapper_.env(), compiler_config_path, -+ cypher_query_string); - #else -- return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, -- graph_statistic_json_, compiler_config_path, -- cypher_query_string); -+ return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, -+ graph_statistic_json_, compiler_config_path, -+ cypher_query_string); - #endif --} -+ } - - } // namespace gs -diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h -index 58ef9c7797..5c1229ef29 100644 ---- a/interactive_engine/compiler/src/main/native/graph_planner.h -+++ b/interactive_engine/compiler/src/main/native/graph_planner.h -@@ -29,119 +29,131 @@ limitations under the License. - #define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess - #endif - --namespace gs { -+namespace gs -+{ - --struct Plan { -- physical::PhysicalPlan physical_plan; -- std::string result_schema; --}; -+ struct Plan -+ { -+ physical::PhysicalPlan physical_plan; -+ std::string result_schema; -+ }; - - #if (GRAPH_PLANNER_JNI_INVOKER) --namespace jni { --struct JNIEnvMark { -- JNIEnv *_env; -- -- JNIEnvMark(); -- JNIEnvMark(const std::string &jvm_options); -- ~JNIEnvMark(); -- JNIEnv *env(); --}; -- --} // namespace jni -+ namespace jni -+ { -+ struct JNIEnvMark -+ { -+ JNIEnv *_env; -+ -+ JNIEnvMark(); -+ JNIEnvMark(const std::string &jvm_options); -+ ~JNIEnvMark(); -+ JNIEnv *env(); -+ }; -+ -+ } // namespace jni - #endif - --class GraphPlannerWrapper { --public: -- static constexpr const char *kGraphPlannerClass = -- "com/alibaba/graphscope/common/ir/tools/GraphPlanner"; -- static constexpr const char *GRAPH_PLANNER_FULL_NAME = -- "com.alibaba.graphscope.common.ir.tools.GraphPlanner"; -- static constexpr const char *kGraphPlannerMethod = "generatePhysicalPlan"; -- static constexpr const char *kGraphPlannerMethodSignature = -- "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; -- -- GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, -- const std::string &graph_schema_yaml, -- const std::string &graph_statistic_json = "") -+ class GraphPlannerWrapper -+ { -+ public: -+ static constexpr const char *kGraphPlannerClass = -+ "com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI"; -+ static constexpr const char *GRAPH_PLANNER_FULL_NAME = -+ "com.alibaba.graphscope.common.jna._native.GraphPlannerJNI"; -+ static constexpr const char *kGraphPlannerMethod = "compilePlan"; -+ static constexpr const char *kGraphPlannerMethodSignature = -+ "(Ljava/lang/String;Ljava/lang/String;)[Ljava/lang/Object;"; -+ -+ GraphPlannerWrapper(const std::string java_path, const std::string &jna_path, -+ const std::string &graph_schema_yaml, -+ const std::string &graph_statistic_json = "") - #if (GRAPH_PLANNER_JNI_INVOKER) -- : jni_wrapper_(generate_jvm_options( -- java_path, jna_path, graph_schema_yaml, graph_statistic_json)) { -- jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); -- if (clz == NULL) { -- std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; -- return; -+ : jni_wrapper_(generate_jvm_options( -+ java_path, jna_path, graph_schema_yaml, graph_statistic_json)) -+ { -+ jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); -+ if (clz == NULL) -+ { -+ std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; -+ return; -+ } -+ graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); -+ jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( -+ graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); -+ if (j_method_id == NULL) -+ { -+ std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; -+ return; -+ } -+ graph_planner_method_id_ = j_method_id; - } -- graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); -- jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( -- graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); -- if (j_method_id == NULL) { -- std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; -- return; -- } -- graph_planner_method_id_ = j_method_id; -- } - #else -- : jna_path_("-Djna.library.path=" + jna_path), -- graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), -- graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) { -- class_path_ = expand_directory(java_path); -- } -+ : jna_path_("-Djna.library.path=" + jna_path), -+ graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), -+ graph_statistic_json_("-Dgraph.statistic=" + graph_statistic_json) -+ { -+ class_path_ = expand_directory(java_path); -+ } - #endif - -- ~GraphPlannerWrapper() { -+ ~GraphPlannerWrapper() -+ { - #if (GRAPH_PLANNER_JNI_INVOKER) -- if (graph_planner_clz_ != NULL) { -- jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); -- } -+ if (graph_planner_clz_ != NULL) -+ { -+ jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); -+ } - #endif -- } -+ } - -- inline bool is_valid() { -+ inline bool is_valid() -+ { - #if (GRAPH_PLANNER_JNI_INVOKER) -- return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; -+ return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; - #else -- return true; // just return true, since we don't have a way to check the -- // validity when calling via subprocess. -+ return true; // just return true, since we don't have a way to check the -+ // validity when calling via subprocess. - #endif -- } -- -- /** -- * @brief Invoker GraphPlanner to generate a physical plan from a cypher -- * query. -- * @param compiler_config_path The path of compiler config file. -- * @param cypher_query_string The cypher query string. -- * @return physical plan in string. -- */ -- Plan CompilePlan(const std::string &compiler_config_path, -- const std::string &cypher_query_string); -- --private: -- std::string generate_jvm_options(const std::string java_path, -- const std::string &jna_path, -- const std::string &graph_schema_yaml, -- const std::string &graph_statistic_json); -- // physical::PhysicalPlan compilePlanJNI(const std::string& -- // compiler_config_path, -- // const std::string& -- // cypher_query_string); -- std::string expand_directory(const std::string &path); -+ } -+ -+ /** -+ * @brief Invoker GraphPlanner to generate a physical plan from a cypher -+ * query. -+ * @param compiler_config_path The path of compiler config file. -+ * @param cypher_query_string The cypher query string. -+ * @return physical plan in string. -+ */ -+ Plan CompilePlan(const std::string &compiler_config_path, -+ const std::string &cypher_query_string); -+ -+ private: -+ std::string generate_jvm_options(const std::string java_path, -+ const std::string &jna_path, -+ const std::string &graph_schema_yaml, -+ const std::string &graph_statistic_json); -+ // physical::PhysicalPlan compilePlanJNI(const std::string& -+ // compiler_config_path, -+ // const std::string& -+ // cypher_query_string); -+ std::string expand_directory(const std::string &path); - #if (GRAPH_PLANNER_JNI_INVOKER) -- // We need to list all files in the directory, if exists. -- // The reason why we need to list all files in the directory is that -- // java -Djava.class.path=dir/* (in jni, which we are using)will not load all -- // jar files in the directory, While java -cp dir/* will load all jar files in -- // the directory. -- -- gs::jni::JNIEnvMark jni_wrapper_; -- jclass graph_planner_clz_; -- jmethodID graph_planner_method_id_; -+ // We need to list all files in the directory, if exists. -+ // The reason why we need to list all files in the directory is that -+ // java -Djava.class.path=dir/* (in jni, which we are using)will not load all -+ // jar files in the directory, While java -cp dir/* will load all jar files in -+ // the directory. -+ -+ gs::jni::JNIEnvMark jni_wrapper_; -+ jclass graph_planner_clz_; -+ jmethodID graph_planner_method_id_; - #else -- std::string class_path_; -- std::string jna_path_; -- std::string graph_schema_yaml_; -- std::string graph_statistic_json_; -+ std::string class_path_; -+ std::string jna_path_; -+ std::string graph_schema_yaml_; -+ std::string graph_statistic_json_; - #endif --}; -+ }; - } // namespace gs - - #endif // PLANNER_GRAPH_PLANNER_H_ -diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java -index 5469e8ad55..267363aa8a 100644 ---- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java -+++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java -@@ -18,21 +18,21 @@ - - package com.alibaba.graphscope.common.ir; - --import com.alibaba.graphscope.common.ir.tools.GraphPlanner; -+import com.alibaba.graphscope.common.jna._native.GraphPlannerJNI; -+import com.alibaba.graphscope.common.jna._native.JNIPlan; - import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; -- - import org.junit.Test; - - public class JNITest { - @Test - public void test() throws Exception { -- Object[] objects = -- GraphPlanner.generatePhysicalPlan( -+ JNIPlan objects = -+ GraphPlannerJNI.compilePlan( - "conf/ir.compiler.properties", "Match (n) Return n, count(n)"); - GraphAlgebraPhysical.PhysicalPlan plan = -- GraphAlgebraPhysical.PhysicalPlan.parseFrom((byte[]) objects[0]); -+ GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); - System.out.println(plan); -- String resultSchema = (String) objects[1]; -+ String resultSchema = objects.resultSchemaYaml; - System.out.println(resultSchema); - } - } diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index b93fa160bdc7..0f88b8a48834 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -171,6 +171,10 @@ ch.qos.logback logback-classic + + org.springframework.boot + spring-boot-starter-web + diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java similarity index 77% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java index 689c9c74efc5..69820fad8964 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/JNIPlan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java @@ -16,13 +16,15 @@ * */ -package com.alibaba.graphscope.common.jna._native; +package com.alibaba.graphscope.sdk; -public class JNIPlan { - public final byte[] physicalBytes; - public final String resultSchemaYaml; +import java.io.Serializable; - public JNIPlan(byte[] physicalBytes, String resultSchemaYaml) { +public class GraphPlan implements Serializable { + private final byte[] physicalBytes; + public String resultSchemaYaml; + + public GraphPlan(byte[] physicalBytes, String resultSchemaYaml) { this.physicalBytes = physicalBytes; this.resultSchemaYaml = resultSchemaYaml; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java similarity index 95% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java index 3a34facb2a24..a048b9161d2f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java @@ -16,7 +16,7 @@ * */ -package com.alibaba.graphscope.common.jna._native; +package com.alibaba.graphscope.sdk; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.meta.GraphId; @@ -41,7 +41,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -public class GraphPlannerJNI { +public class PlanUtils { /** * Provide a java-side implementation to compile the query in string to a physical plan * @param configPath @@ -52,7 +52,7 @@ public class GraphPlannerJNI { * resultSchemaYaml defines the result specification of the query in yaml format * @throws Exception */ - public static JNIPlan compilePlan( + public static GraphPlan compilePlan( String configPath, String query, String schemaYaml, String statsJson) throws Exception { GraphPlanner.Summary summary = GraphPlanner.generatePlan( @@ -71,7 +71,7 @@ public static JNIPlan compilePlan( logicalPlan.getDynamicParams()); ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); - return new JNIPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); + return new GraphPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); } static class StringMetaReader implements IrMetaReader { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java new file mode 100644 index 000000000000..1dc7274ef9f4 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java @@ -0,0 +1,91 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.examples; + +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class TestGraphPlanner { + public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); + } + + private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); + } + + // get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan + private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); + } + + // get result schema yaml from json + private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java new file mode 100644 index 000000000000..eb82798fba87 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java @@ -0,0 +1,69 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import java.io.Serializable; + +public class GraphPlanRequest implements Serializable { + private String configPath; + private String query; + private String schemaYaml; + private String statsJson; + + public GraphPlanRequest() {} + + public GraphPlanRequest(String configPath, String query, String schemaYaml, String statsJson) { + this.configPath = configPath; + this.query = query; + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + } + + public String getConfigPath() { + return configPath; + } + + public String getQuery() { + return query; + } + + public String getSchemaYaml() { + return schemaYaml; + } + + public String getStatsJson() { + return statsJson; + } + + public void setConfigPath(String configPath) { + this.configPath = configPath; + } + + public void setQuery(String query) { + this.query = query; + } + + public void setSchemaYaml(String schemaYaml) { + this.schemaYaml = schemaYaml; + } + + public void setStatsJson(String statsJson) { + this.statsJson = statsJson; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java new file mode 100644 index 000000000000..aad9d3d01ba1 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; + +import java.io.Serializable; +import java.util.Objects; + +public class GraphPlanResponse implements Serializable { + private GraphPlan graphPlan; + private String errorMessage; + + public GraphPlanResponse() {} + + public GraphPlanResponse(GraphPlan graphPlan) { + this.graphPlan = Objects.requireNonNull(graphPlan); + this.errorMessage = null; + } + + public GraphPlanResponse(String errorMessage) { + this.graphPlan = null; + this.errorMessage = Objects.requireNonNull(errorMessage); + } + + public GraphPlan getGraphPlan() { + return graphPlan; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setGraphPlan(GraphPlan graphPlan) { + this.graphPlan = graphPlan; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java new file mode 100644 index 000000000000..4cacf9e893c5 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java @@ -0,0 +1,54 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; +import com.alibaba.graphscope.sdk.PlanUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +@RequestMapping("/api") +public class GraphPlannerController { + private final Logger logger = LoggerFactory.getLogger(GraphPlannerController.class); + + @PostMapping("/compilePlan") + public ResponseEntity compilePlan(@RequestBody GraphPlanRequest request) { + try { + GraphPlan plan = + PlanUtils.compilePlan( + request.getConfigPath(), + request.getQuery(), + request.getSchemaYaml(), + request.getStatsJson()); + return ResponseEntity.ok(new GraphPlanResponse(plan)); + } catch (Exception e) { + logger.error("Failed to compile plan", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new GraphPlanResponse(e.getMessage())); + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java new file mode 100644 index 000000000000..7e2b9745d906 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java @@ -0,0 +1,29 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class GraphPlannerService { + public static void main(String[] args) { + SpringApplication.run(GraphPlannerService.class, args); + } +} diff --git a/interactive_engine/compiler/src/main/native/README.md b/interactive_engine/compiler/src/main/native/README.md index 0d8d4b834912..f278ffe9433b 100644 --- a/interactive_engine/compiler/src/main/native/README.md +++ b/interactive_engine/compiler/src/main/native/README.md @@ -87,3 +87,135 @@ Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, const std::string &graph_schema_yaml, const std::string &graph_statistic_json) ``` + +Here is a refined version of your documentation with improvements for clarity, consistency, and readability: + +## Restful API + +We provide an alternative method to expose the interface as a RESTful API. Follow the steps below to access the interface via REST. + +### Step 1: Build the Project + +To build the project, run the following command: +```bash +cd interactive_engine +# Use '-Dskip.native=true' to skip compiling C++ native code +mvn clean package -DskipTests -Pgraph-planner-jni -Dskip.native=true +``` + +### Step 2: Locate and Extract the Package + +Once the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents as follows: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +### Step 3: Start the Graph Planner RESTful Service + +To start the service, run the following command: + +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.restful.GraphPlannerService --spring.config.location=./conf/application.yaml +``` + +### Step 4: Access the RESTful API + +To send a request to the RESTful API, use the following `curl` command: + +```bash +curl -X POST http://localhost:8080/api/compilePlan \ + -H "Content-Type: application/json" \ + -d "{ + \"configPath\": \"$configPath\", + \"query\": \"$query\", + \"schemaYaml\": \"$schemaYaml\", + \"statsJson\": \"$statsJson\" + }" +``` + +Replace `$configPath`, `$query`, `$schemaYaml`, and `$statsJson` with the appropriate values. + +The response will be in JSON format, similar to: + +```json +{ + "graphPlan": { + "physicalBytes": "", + "resultSchemaYaml": "" + } +} +``` + +The response contains two fields: +1. physicalBytes: A Base64-encoded string representing the physical plan bytes. +2. resultSchemaYaml: A string representing the YAML schema. + +You can decode these values into the required structures. + +Alternatively, you can run the following `Java` command to execute the same request: + +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/interactive_config_test.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json +``` + +Here’s an example of how to access the RESTful API and decode the response in Java code: +``` +public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); +} + +private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); +} + +// get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan +private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); +} + +// get result schema yaml from json +private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); +} +``` + \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h index 613e9bc9abd5..40de6b3d0161 100644 --- a/interactive_engine/compiler/src/main/native/graph_planner.h +++ b/interactive_engine/compiler/src/main/native/graph_planner.h @@ -58,12 +58,12 @@ namespace gs { public: static constexpr const char *kGraphPlannerClass = - "com/alibaba/graphscope/common/jna/_native/GraphPlannerJNI"; + "com/alibaba/graphscope/sdk/PlanUtils"; static constexpr const char *GRAPH_PLANNER_FULL_NAME = - "com.alibaba.graphscope.common.jna._native.GraphPlannerJNI"; + "com.alibaba.graphscope.sdk.PlanUtils"; static constexpr const char *kGraphPlannerMethod = "compilePlan"; static constexpr const char *kGraphPlannerMethodSignature = - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/common/jna/_native/JNIPlan;"; + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/sdk/GraphPlan;"; /** * @brief Constructs a new GraphPlannerWrapper object diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java index dc0b618d8f79..046c923897cb 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java @@ -18,22 +18,17 @@ package com.alibaba.graphscope.common.ir; -import com.alibaba.graphscope.common.jna._native.GraphPlannerJNI; -import com.alibaba.graphscope.common.jna._native.JNIPlan; -import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; - -import org.junit.Test; - public class JNITest { - @Test - public void test() throws Exception { - JNIPlan objects = - GraphPlannerJNI.compilePlan( - "conf/ir.compiler.properties", "Match (n) Return n, count(n)", "", ""); - GraphAlgebraPhysical.PhysicalPlan plan = - GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); - System.out.println(plan); - String resultSchema = objects.resultSchemaYaml; - System.out.println(resultSchema); - } + // @Test + // public void test() throws Exception { + // GraphPlan objects = + // PlanUtils.compilePlan( + // "conf/ir.compiler.properties", "Match (n) Return n, count(n)", "", + // ""); + // GraphAlgebraPhysical.PhysicalPlan plan = + // GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); + // System.out.println(plan); + // String resultSchema = objects.resultSchemaYaml; + // System.out.println(resultSchema); + // } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java new file mode 100644 index 000000000000..af6d84c91f00 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java @@ -0,0 +1,86 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class GraphPlannerTest { + @Test + public void testCompilePlan() throws Exception { + String jsonPayLoad = createParameters().toString(); + HttpClient client = HttpClient.newBuilder().build(); + System.out.println(jsonPayLoad); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + System.out.println(body); + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); + } + + private JsonNode createParameters() throws Exception { + Map params = + ImmutableMap.of( + "configPath", + "conf/ir.compiler.properties", + "query", + "Match (n) Return n;", + "schemaYaml", + FileUtils.readFileToString( + new File("src/test/resources/config/modern/graph.yaml"), + StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString( + new File("src/test/resources/statistics/modern_statistics.json"), + StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); + } + + private GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); + } + + private String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); + } +} diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index 01519069f73b..f89dd1754e40 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -51,6 +51,9 @@ graph-planner-jni + + false + false @@ -78,6 +81,7 @@ + ${skip.native} @@ -718,6 +722,11 @@ ${interactive.sdk.version} ${interactive.sdk.classifier} + + org.springframework.boot + spring-boot-starter-web + 2.2.4.RELEASE + From f23b478a5aa551e33a3cb908c30c4c04ab726387 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 12 Dec 2024 21:00:49 +0800 Subject: [PATCH 16/27] impl GraphPlanner as single instance to avoid creation overhead in each jni call --- .../graphscope/common/config/YamlConfigs.java | 3 + .../graphscope/sdk/GraphPlanerInstance.java | 63 +++++++++++++++++++ .../com/alibaba/graphscope/sdk/PlanUtils.java | 15 ++++- .../compiler/src/main/native/test/test.cc | 9 +++ .../alibaba/graphscope/common/ir/JNITest.java | 34 ---------- .../graphscope/sdk/JNICompilePlanTest.java | 49 +++++++++++++++ ...nnerTest.java => RestCompilePlanTest.java} | 2 +- 7 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java delete mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java rename interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/{GraphPlannerTest.java => RestCompilePlanTest.java} (99%) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 1670362ccfcc..438d1b22b211 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -37,6 +37,9 @@ public class YamlConfigs extends Configs { "graph.planner.is.on", (Configs configs) -> configs.get("compiler.planner.is_on")) .put("graph.planner.opt", (Configs configs) -> configs.get("compiler.planner.opt")) + .put( + "graph.planner.cbo.glogue.size", + (Configs configs) -> configs.get("compiler.planner.cbo.glogue.size")) .put( "graph.planner.rules", (Configs configs) -> { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java new file mode 100644 index 000000000000..ab5947a8095f --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; +import com.alibaba.graphscope.common.ir.planner.PlannerGroupManager; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlanFactory; + +public class GraphPlanerInstance { + private final GraphPlanner planner; + private final IrMeta meta; + + private static GraphPlanerInstance instance; + + public GraphPlanerInstance(GraphPlanner planner, IrMeta meta) { + this.planner = planner; + this.meta = meta; + } + + public static synchronized GraphPlanerInstance getInstance( + String configPath, GraphPlanner.IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + if (instance == null) { + Configs configs = Configs.Factory.create(configPath); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = + metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + instance = new GraphPlanerInstance(planner, metaFetcher.fetch().get()); + } + return instance; + } + + public GraphPlanner getPlanner() { + return planner; + } + + public IrMeta getMeta() { + return meta; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java index a048b9161d2f..547cadcfc0c5 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java @@ -36,12 +36,16 @@ import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; public class PlanUtils { + private static final Logger logger = LoggerFactory.getLogger(PlanUtils.class); /** * Provide a java-side implementation to compile the query in string to a physical plan * @param configPath @@ -54,13 +58,16 @@ public class PlanUtils { */ public static GraphPlan compilePlan( String configPath, String query, String schemaYaml, String statsJson) throws Exception { - GraphPlanner.Summary summary = - GraphPlanner.generatePlan( + long startTime = System.currentTimeMillis(); + GraphPlanerInstance instance = + GraphPlanerInstance.getInstance( configPath, - query, (Configs configs, IrMetaTracker tracker) -> new StaticIrMetaFetcher( new StringMetaReader(schemaYaml, statsJson), tracker)); + GraphPlanner.PlannerInstance plannerInstance = + instance.getPlanner().instance(query, instance.getMeta()); + GraphPlanner.Summary summary = plannerInstance.plan(); LogicalPlan logicalPlan = summary.getLogicalPlan(); PhysicalPlan physicalPlan = summary.getPhysicalPlan(); StoredProcedureMeta procedureMeta = @@ -71,6 +78,8 @@ public static GraphPlan compilePlan( logicalPlan.getDynamicParams()); ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); + long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("compile plan cost: {} ms", elapsedTime); return new GraphPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); } diff --git a/interactive_engine/compiler/src/main/native/test/test.cc b/interactive_engine/compiler/src/main/native/test/test.cc index f0e161f4a053..1c58f11af6eb 100644 --- a/interactive_engine/compiler/src/main/native/test/test.cc +++ b/interactive_engine/compiler/src/main/native/test/test.cc @@ -16,6 +16,7 @@ limitations under the License. #include #include +#include #include @@ -85,6 +86,8 @@ int main(int argc, char **argv) return 1; } + // auto start = std::chrono::high_resolution_clock::now(); + std::string java_class_path = argv[1]; std::string jna_class_path = argv[2]; std::string graph_schema_path = argv[3]; @@ -106,6 +109,12 @@ int main(int argc, char **argv) std::string config_path = argv[6]; auto plan = graph_planner_wrapper.CompilePlan(config_path, cypher_query_string, schema_content, statistic_content); + + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start).count(); + + // std::cout << "total execution time: " << duration << " ms" << std::endl; + std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; std::cout << "schema: " << plan.result_schema << std::endl; return 0; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java deleted file mode 100644 index 046c923897cb..000000000000 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/JNITest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package com.alibaba.graphscope.common.ir; - -public class JNITest { - // @Test - // public void test() throws Exception { - // GraphPlan objects = - // PlanUtils.compilePlan( - // "conf/ir.compiler.properties", "Match (n) Return n, count(n)", "", - // ""); - // GraphAlgebraPhysical.PhysicalPlan plan = - // GraphAlgebraPhysical.PhysicalPlan.parseFrom(objects.physicalBytes); - // System.out.println(plan); - // String resultSchema = objects.resultSchemaYaml; - // System.out.println(resultSchema); - // } -} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java new file mode 100644 index 000000000000..cadc5dbc5649 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java @@ -0,0 +1,49 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; + +public class JNICompilePlanTest { + @Test + public void planQuery() throws Exception { + String configPath = "src/test/resources/config/interactive_config_test.yaml"; + String query = + "MATCH (src)-[e:test6*4..5]->(dest) WHERE src.__domain__ = 'xzz' RETURN" + + " src.__entity_id__ AS sId, dest.__entity_id__ AS dId;"; + String schemaYaml = + FileUtils.readFileToString( + new File("/Users/zhouxiaoli/Downloads/graph_schema.yaml"), + StandardCharsets.UTF_8); + String statsJson = + FileUtils.readFileToString( + new File("/Users/zhouxiaoli/Downloads/statistics.json"), + StandardCharsets.UTF_8); + for (int i = 0; i < 100; ++i) { + long startTime = System.currentTimeMillis(); + PlanUtils.compilePlan(configPath, query, schemaYaml, statsJson); + long endTime = System.currentTimeMillis() - startTime; + System.out.println("Time cost: " + endTime); + } + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java similarity index 99% rename from interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java rename to interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java index af6d84c91f00..be5418d2141a 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/GraphPlannerTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java @@ -34,7 +34,7 @@ import java.nio.charset.StandardCharsets; import java.util.Map; -public class GraphPlannerTest { +public class RestCompilePlanTest { @Test public void testCompilePlan() throws Exception { String jsonPayLoad = createParameters().toString(); From 0c6f3fd66b8b3aa0c8d6d01d1f2350964dce0d98 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 8 Jan 2025 17:53:11 +0800 Subject: [PATCH 17/27] add function uri in yaml configs --- .../alibaba/graphscope/common/config/YamlConfigs.java | 9 +++++++++ .../alibaba/graphscope/common/config/YamlConfigTest.java | 3 +++ .../test/resources/config/gs_interactive_hiactor.yaml | 2 ++ 3 files changed, 14 insertions(+) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 438d1b22b211..cca6b383c6cf 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -68,6 +68,15 @@ public class YamlConfigs extends Configs { } return configs.get("compiler.meta.reader.statistics.uri"); }) + .put( + "graph.functions", + (Configs configs) -> { + String functions = System.getProperty("graph.functions"); + if (functions != null) { + return functions; + } + return configs.get("compiler.meta.reader.functions.uri"); + }) .put( "graph.meta.schema.fetch.interval.ms", (Configs configs) -> configs.get("compiler.meta.reader.schema.interval")) diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java index 4914e682ad21..c7f424504a47 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java @@ -100,5 +100,8 @@ public void compiler_config_test() throws Exception { YamlConfigs configs = new YamlConfigs("config/gs_interactive_hiactor.yaml", FileLoadType.RESOURCES); Assert.assertEquals("UTF-8", FrontendConfig.CALCITE_DEFAULT_CHARSET.get(configs)); + Assert.assertEquals( + "./src/main/resources/conf/graph_functions.yaml", + GraphConfig.GRAPH_FUNCTIONS_URI.get(configs)); } } diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml index bc703b2d32ff..213dc5a3b782 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml @@ -18,6 +18,8 @@ compiler: reader: schema: uri: ./target/test-classes/config/modern/graph.yaml + functions: + uri: ./src/main/resources/conf/graph_functions.yaml endpoint: default_listen_address: 0.0.0.0 # default localhost bolt_connector: # for cypher, there may be other connectors, such as bolt_connector, https_connector From 85855b9d0a00827ba958427b48152bbf17553305 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 8 Jan 2025 18:00:59 +0800 Subject: [PATCH 18/27] minor fix --- .../graphscope/common/ir/meta/function/GraphFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java index 21374d84b07d..863bc6230b8a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java @@ -27,6 +27,8 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; import java.io.ByteArrayInputStream; @@ -38,6 +40,7 @@ import java.util.Map; public class GraphFunctions { + private static final Logger logger = LoggerFactory.getLogger(GraphFunctions.class); public static final String FUNCTION_PREFIX = "gs.function."; private final Map functionMetaMap; @@ -85,7 +88,7 @@ private void registerConfigFunctions(Configs configs) { } } } catch (Exception e) { - throw new RuntimeException(e); + logger.warn("fail to load graph functions from configuration", e); } } From e34049ad8a55b09dcdbe3770039ed5bbd799cf22 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 9 Jan 2025 20:04:21 +0800 Subject: [PATCH 19/27] add docs for graph planner interface --- docs/index.rst | 1 + .../interactive_engine/graph_planner.md | 161 ++++++++++-------- 2 files changed, 88 insertions(+), 74 deletions(-) rename interactive_engine/compiler/src/main/native/README.md => docs/interactive_engine/graph_planner.md (64%) diff --git a/docs/index.rst b/docs/index.rst index b50a209bc91f..86d94a9228ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -74,6 +74,7 @@ and the vineyard store that offers efficient in-memory data transfers. interactive_engine/tinkerpop_eco interactive_engine/neo4j_eco interactive_engine/gopt + interactive_engine/graph_planner interactive_engine/benchmark_tool .. interactive_engine/guide_and_examples interactive_engine/design_of_gie diff --git a/interactive_engine/compiler/src/main/native/README.md b/docs/interactive_engine/graph_planner.md similarity index 64% rename from interactive_engine/compiler/src/main/native/README.md rename to docs/interactive_engine/graph_planner.md index f278ffe9433b..f5de1454e778 100644 --- a/interactive_engine/compiler/src/main/native/README.md +++ b/docs/interactive_engine/graph_planner.md @@ -1,77 +1,33 @@ -# Using the Graph Planner JNI Interface for C++ Invocation +# Graph Planner Interface by JNI and RESTful API -Follow the steps below to get started with the Graph Planner JNI interface for c++ invocation. +`GraphPlanner` is the primary entry point for GOpt’s query optimization and physical plan generation. Originally, it was tightly integrated into the Frontend service, where it optimized Cypher queries received via the Bolt protocol and generated execution plans for various backend engines. -## Getting Started -### Step 1: Build the Project +To enhance its flexibility and ease of integration, `GraphPlanner` is now available as a standalone module, free from any dependencies on other Frontend modules. It supports both JNI and RESTful API interfaces, enabling lightweight and straightforward integration into diverse systems. Whether you are working on a native application or web-based services, `GraphPlanner` can seamlessly integrate into your architecture, providing efficient query optimization and physical plan generation across a wide range of use cases. -Navigate to the project directory and build the package using Maven: -```bash -cd interactive_engine -mvn clean package -DskipTests -Pgraph-planner-jni -``` - -### Step 2: Locate and Extract the Package - -After the build completes, a tarball named `graph-planner-jni.tar.gz` will be available in the `assembly/target` directory. Extract the contents of the tarball: - -```bash -cd assembly/target -tar xvzf graph-planner-jni.tar.gz -cd graph-planner-jni -``` - -### Step 3: Run the Example Binary -To demonstrate the usage of the JNI interface, an example binary `test_graph_planner` is provided. Use the following command to execute it: +## JNI API -```bash -# bin/test_graph_planner -bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/interactive_config_test.yaml -``` +### Interface Overview -The output includes the physical plan and result schema in YAML format. Below is an example of a result schema: - -```yaml -schema: - name: default - description: default desc - mode: READ - extension: .so - library: libdefault.so - params: [] -returns: - - name: n - type: {primitive_type: DT_UNKNOWN} - - name: $f1 - type: {primitive_type: DT_SIGNED_INT64} -type: UNKNOWN -query: MATCH (n) RETURN n, COUNT(n); -``` +We provide a c++ wrapper implementation `GraphPlannerWrapper` for the JNI interface. Here is a brief explanation of the logical interface provided by the `c++` class. -The `returns` field defines the result schema. Each entry’s name specifies the column name, and the order of each entry determines the column IDs. - -## Explanation - -Below is a brief explanation of the interface provided by the example: - -### Constructor +Constructor: ```cpp -/** - * @brief Constructs a new GraphPlannerWrapper object - * @param java_path Java class path - * @param jna_path JNA library path - * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) +/** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) - */ -GraphPlannerWrapper(const std::string &java_path, - const std::string &jna_path, - const std::string &graph_schema_yaml = "", - const std::string &graph_statistic_json = ""); + */ +GraphPlannerWrapper(const std::string &java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml = "", + const std::string &graph_statistic_json = ""); ``` -## Method +Method: ```cpp /** @@ -88,13 +44,69 @@ Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, const std::string &graph_statistic_json) ``` -Here is a refined version of your documentation with improvements for clarity, consistency, and readability: +### Getting Started +Follow the steps below to get started with the Graph Planner interface for c++ invocation. + +#### Step 1: Build the Project + +Navigate to the project directory and build the package using Maven: +```bash +cd interactive_engine +mvn clean package -DskipTests -Pgraph-planner-jni +``` + +#### Step 2: Locate and Extract the Package + +After the build completes, a tarball named `graph-planner-jni.tar.gz` will be available in the `assembly/target` directory. Extract the contents of the tarball: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +#### Step 3: Run the Example Binary + +To demonstrate the usage of the JNI interface, an example binary `test_graph_planner` is provided. Use the following command to execute it: + +```bash +# bin/test_graph_planner +bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/interactive_config_test.yaml +``` + +The output consists of the physical plan (in byte format) and the result schema (in YAML format). The physical plan adheres to the specifications defined in the [protobuf](). + +Below is an example of a result schema: + +```yaml +schema: + name: default + description: default desc + mode: READ + extension: .so + library: libdefault.so + params: [] +returns: + - name: n + type: {primitive_type: DT_UNKNOWN} + - name: $f1 + type: {primitive_type: DT_SIGNED_INT64} +type: UNKNOWN +query: MATCH (n) RETURN n, COUNT(n); +``` + +The `returns` field defines the structure of the data returned by backend engines. Each nested entry in the returns field includes three components: +- the column name, which specifies the name of the result column; +- the entry’s ordinal position, which determines the column ID; +- the type, which enforces the data type constraint for the column. ## Restful API We provide an alternative method to expose the interface as a RESTful API. Follow the steps below to access the interface via REST. -### Step 1: Build the Project +### Getting Started + +#### Step 1: Build the Project To build the project, run the following command: ```bash @@ -103,7 +115,7 @@ cd interactive_engine mvn clean package -DskipTests -Pgraph-planner-jni -Dskip.native=true ``` -### Step 2: Locate and Extract the Package +#### Step 2: Locate and Extract the Package Once the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents as follows: @@ -113,7 +125,7 @@ tar xvzf graph-planner-jni.tar.gz cd graph-planner-jni ``` -### Step 3: Start the Graph Planner RESTful Service +#### Step 3: Start the Graph Planner RESTful Service To start the service, run the following command: @@ -121,7 +133,7 @@ To start the service, run the following command: java -cp ".:./libs/*" com.alibaba.graphscope.sdk.restful.GraphPlannerService --spring.config.location=./conf/application.yaml ``` -### Step 4: Access the RESTful API +#### Step 4: Access the RESTful API by `Curl` To send a request to the RESTful API, use the following `curl` command: @@ -155,14 +167,11 @@ The response contains two fields: You can decode these values into the required structures. -Alternatively, you can run the following `Java` command to execute the same request: +#### Step 4: Access the RESTful API by `Java` Sdk -```bash -java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/interactive_config_test.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json -``` +Alternatively, if you are a java-side user, we provide a java sdk example to guide you how to access the restful API and decode the response : -Here’s an example of how to access the RESTful API and decode the response in Java code: -``` +```java public static void main(String[] args) throws Exception { if (args.length < 4) { System.out.println("Usage: "); @@ -218,4 +227,8 @@ private static String getResultSchemaYaml(JsonNode planNode) { return planNode.get("resultSchemaYaml").asText(); } ``` - \ No newline at end of file + +Run the java sdk example with the following command: +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/interactive_config_test.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json +``` \ No newline at end of file From f8e2140e6af7e83886af5f1d6c38c0bf8b511b81 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 9 Jan 2025 20:22:32 +0800 Subject: [PATCH 20/27] revert changes to parent pom --- interactive_engine/pom.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index f913334628e5..b6dcfeb464ff 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -91,6 +91,23 @@ + + graphscope + + true + + + v6d + + + assembly + common + executor + frontend + executor/engine/pegasus/clients/java/client + compiler + + groot From eb4edc3f3bb7a957b30dea780583c9e11a384343 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 10 Jan 2025 10:38:10 +0800 Subject: [PATCH 21/27] revert changes to compiler conf --- interactive_engine/compiler/conf/ir.compiler.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index eb13aa78912f..2ab6058f579c 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json graph.store: exp graph.planner.is.on: true -graph.planner.opt: CBO +graph.planner.opt: RBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule # set statistics access uri -graph.statistics: src/test/resources/statistics/modern_statistics.json +# graph.statistics: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: @@ -60,7 +60,7 @@ calcite.default.charset: UTF-8 # gremlin.script.language.name: antlr_gremlin_traversal # the output plan format, can be ffi(default) or proto -graph.physical.opt: proto +# graph.physical.opt: ffi # set the max capacity of the result streaming buffer for each query # per.query.stream.buffer.max.capacity: 256 From f251452c2e0281e4bc682e3758680c8f4d6a49f3 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 10 Jan 2025 10:46:50 +0800 Subject: [PATCH 22/27] fix hiactor config used for graph planner jni examples --- docs/interactive_engine/graph_planner.md | 4 ++-- interactive_engine/assembly/graph-planner-jni.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/interactive_engine/graph_planner.md b/docs/interactive_engine/graph_planner.md index f5de1454e778..edef5ff9a16c 100644 --- a/docs/interactive_engine/graph_planner.md +++ b/docs/interactive_engine/graph_planner.md @@ -71,7 +71,7 @@ To demonstrate the usage of the JNI interface, an example binary `test_graph_pla ```bash # bin/test_graph_planner -bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/interactive_config_test.yaml +bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/gs_interactive_hiactor.yaml ``` The output consists of the physical plan (in byte format) and the result schema (in YAML format). The physical plan adheres to the specifications defined in the [protobuf](). @@ -230,5 +230,5 @@ private static String getResultSchemaYaml(JsonNode planNode) { Run the java sdk example with the following command: ```bash -java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/interactive_config_test.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/gs_interactive_hiactor.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json ``` \ No newline at end of file diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml index 3413eed2a566..d77e506d51b6 100644 --- a/interactive_engine/assembly/graph-planner-jni.xml +++ b/interactive_engine/assembly/graph-planner-jni.xml @@ -65,7 +65,7 @@ ${project.parent.basedir}/compiler/src/test/resources/config/ conf - interactive_config_test.yaml + gs_interactive_hiactor.yaml From f8b406a2d19415616a586c42e066e953f798c7d2 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Fri, 10 Jan 2025 15:29:25 +0800 Subject: [PATCH 23/27] revert space Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flex/tests/CMakeLists.txt b/flex/tests/CMakeLists.txt index 4c851c54b9a4..e2bb952d7eb9 100644 --- a/flex/tests/CMakeLists.txt +++ b/flex/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_subdirectory(hqps) -add_subdirectory(rt_mutable_graph) \ No newline at end of file +add_subdirectory(rt_mutable_graph) From 8e7fcdcbb0ad93fd6da8c093e6a902048e93b450 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 10 Jan 2025 15:46:33 +0800 Subject: [PATCH 24/27] fix ci bugs of gs interactive --- .../src/main/java/com/alibaba/graphscope/GraphServer.java | 4 ++-- .../com/alibaba/graphscope/common/config/YamlConfigs.java | 2 +- .../common/ir/meta/fetcher/StaticIrMetaFetcher.java | 2 +- .../common/ir/meta/procedure/StoredProcedureMeta.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index 9a28564c1812..ab11c9b198b2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -210,8 +210,8 @@ private static GraphProperties getTestGraph(Configs configs) { case "rust-mcsr": testGraph = TestGraphFactory.RUST_MCSR; break; - case "native-mcsr": - logger.info("using native-mcsr as test graph"); + case "cpp-mcsr": + logger.info("using cpp-mcsr as test graph"); testGraph = TestGraphFactory.CPP_MCSR; break; default: diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 730183c6246d..9669d119b5c2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -93,7 +93,7 @@ public class YamlConfigs extends Configs { if (configs.get("compute_engine.store.type") != null) { return configs.get("compute_engine.store.type"); } else { - return "native-mcsr"; + return "cpp-mcsr"; } }) .put( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java index 2afd866ad484..b48cc07d81a1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java @@ -51,7 +51,7 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throw try { return this.reader.readStats(meta.getGraphId()); } catch (Exception e) { - logger.warn("failed to read graph statistics, error is: " + e); + logger.warn("failed to read graph statistics, error is {}", e); return null; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index acef051a480a..2963ecb0ee3e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -363,7 +363,7 @@ public static class Config { // option configurations. public static final com.alibaba.graphscope.common.config.Config TYPE = com.alibaba.graphscope.common.config.Config.stringConfig( - "type", "UNKNOWN"); // cypher or native + "type", "UNKNOWN"); // cypher or cpp public static final com.alibaba.graphscope.common.config.Config QUERY = com.alibaba.graphscope.common.config.Config.stringConfig("query", "UNKNOWN"); } From ecfacd43db0a0b5d18e9661c6cbc150c4a85e8e2 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 10 Jan 2025 15:58:06 +0800 Subject: [PATCH 25/27] add basic type proto in CMakeLists --- interactive_engine/compiler/src/main/native/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/interactive_engine/compiler/src/main/native/CMakeLists.txt b/interactive_engine/compiler/src/main/native/CMakeLists.txt index e4157c2aa8ae..d8f0cefd8484 100644 --- a/interactive_engine/compiler/src/main/native/CMakeLists.txt +++ b/interactive_engine/compiler/src/main/native/CMakeLists.txt @@ -42,6 +42,7 @@ set(CODE_GEN_PROTOBUF_FILES ${GIE_COMPILER_PROTO_DIR}/schema.proto ${GIE_COMPILER_PROTO_DIR}/type.proto ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto + ${GIE_COMPILER_PROTO_DIR}/basic_type.proto ) #create directory first From cc6dfc144767cede528e08af86b2626b71902c92 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 10 Jan 2025 17:27:28 +0800 Subject: [PATCH 26/27] fix bugs of reading query in graph planner --- .../graphscope/common/ir/tools/GraphPlanner.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index bc4ef8184a88..090d8f3d3a76 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -267,21 +267,12 @@ public static void main(String[] args) throws Exception { + " 'optional '"); } - BufferedReader reader = new BufferedReader(new FileReader(args[1])); - StringBuilder builder = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - builder.append(line); - } - String query = builder.toString(); - reader.close(); + String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); Summary summary = generatePlan(args[0], query, IrMetaFetcherFactory.DEFAULT); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; - FileOutputStream fos = new FileOutputStream(args[2]); - fos.write(physicalPlan.getContent()); - fos.close(); + FileUtils.writeByteArrayToFile(new File(args[2]), physicalPlan.getContent()); // write stored procedure meta to file LogicalPlan logicalPlan = summary.getLogicalPlan(); From 434754690f41acb3c5b21d1f2a23d35da6084cb5 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Mon, 13 Jan 2025 09:56:01 +0800 Subject: [PATCH 27/27] fixing workflow Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- .github/workflows/interactive.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index c827ffacc8e0..b670081aef5d 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -316,7 +316,7 @@ jobs: - name: Test physical plan generation run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler + cd ${GITHUB_WORKSPACE}/interactive_engine cat > /tmp/physical_plan_gen_config.yaml <> /tmp/physical_plan_gen_config.yaml echo " meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml mvn clean install -DskipTests -Pgraph-planner-jni - ./target/native/test_graph_planner + INTERACTIVE_ENGINE_HOME=${GITHUB_WORKSPACE}/interactive_engine + ./target/native/test_graph_planner ${INTERACTIVE_ENGINE_HOME}/compiler/target/compiler-0.0.1-SNAPSHOT.jar:${INTERACTIVE_ENGINE_HOME}/compiler/target/libs/ ${INTERACTIVE_ENGINE_HOME}/executor/ir/target/release/libir_core.so \ + ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_ENGINE_HOME}/compiler/src/test/resources/statistics/modern_statistics.json \ + "MATCH(n) return count(n);" /tmp/physical_plan_gen_config.yaml - name: Run End-to-End cypher adhoc ldbc query test env: