diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 3d3660c081f5..b670081aef5d 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -314,6 +314,28 @@ jobs: cd ${GITHUB_WORKSPACE}/interactive_engine/ mvn clean install -Pexperimental -DskipTests -q + - name: Test physical plan generation + run: | + cd ${GITHUB_WORKSPACE}/interactive_engine + cat > /tmp/physical_plan_gen_config.yaml <> /tmp/physical_plan_gen_config.yaml + echo " meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml + mvn clean install -DskipTests -Pgraph-planner-jni + INTERACTIVE_ENGINE_HOME=${GITHUB_WORKSPACE}/interactive_engine + ./target/native/test_graph_planner ${INTERACTIVE_ENGINE_HOME}/compiler/target/compiler-0.0.1-SNAPSHOT.jar:${INTERACTIVE_ENGINE_HOME}/compiler/target/libs/ ${INTERACTIVE_ENGINE_HOME}/executor/ir/target/release/libir_core.so \ + ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_ENGINE_HOME}/compiler/src/test/resources/statistics/modern_statistics.json \ + "MATCH(n) return count(n);" /tmp/physical_plan_gen_config.yaml + - name: Run End-to-End cypher adhoc ldbc query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/analytical_engine/apps/pregel/louvain/louvain_app_base.h b/analytical_engine/apps/pregel/louvain/louvain_app_base.h index aa8fd9d53138..8c4df0867759 100644 --- a/analytical_engine/apps/pregel/louvain/louvain_app_base.h +++ b/analytical_engine/apps/pregel/louvain/louvain_app_base.h @@ -223,11 +223,11 @@ class LouvainAppBase auto actual_quality = ctx.compute_context().template get_aggregated_value( actual_quality_aggregator); + assert(actual_quality <= 1.0); // after one pass if already decided halt, that means the pass yield no // changes, so we halt computation. if (current_super_step <= 14 || - std::fabs(actual_quality - ctx.prev_quality()) < - min_quality_improvement) { + (actual_quality - ctx.prev_quality()) <= min_quality_improvement) { // turn to sync community result ctx.compute_context().set_superstep(sync_result_step); syncCommunity(frag, ctx, messages); diff --git a/analytical_engine/apps/pregel/louvain/louvain_vertex.h b/analytical_engine/apps/pregel/louvain/louvain_vertex.h index 3f04e1a30433..c6471e49fbbd 100644 --- a/analytical_engine/apps/pregel/louvain/louvain_vertex.h +++ b/analytical_engine/apps/pregel/louvain/louvain_vertex.h @@ -68,7 +68,7 @@ class LouvainVertex : public PregelVertex { size_t edge_size() { if (!this->use_fake_edges()) { - return this->incoming_edges().Size() + this->outgoing_edges().Size(); + return this->outgoing_edges().Size(); } else { return this->fake_edges().size(); } @@ -88,11 +88,6 @@ class LouvainVertex : public PregelVertex { edata_t get_edge_value(const vid_t& dst_id) { if (!this->use_fake_edges()) { - for (auto& edge : this->incoming_edges()) { - if (this->fragment_->Vertex2Gid(edge.get_neighbor()) == dst_id) { - return static_cast(edge.get_data()); - } - } for (auto& edge : this->outgoing_edges()) { if (this->fragment_->Vertex2Gid(edge.get_neighbor()) == dst_id) { return static_cast(edge.get_data()); @@ -117,12 +112,6 @@ class LouvainVertex : public PregelVertex { edata_t get_edge_values(const std::set& dst_ids) { edata_t ret = 0; if (!this->use_fake_edges()) { - for (auto& edge : this->incoming_edges()) { - auto gid = this->fragment_->Vertex2Gid(edge.get_neighbor()); - if (dst_ids.find(gid) != dst_ids.end()) { - ret += static_cast(edge.get_data()); - } - } for (auto& edge : this->outgoing_edges()) { auto gid = this->fragment_->Vertex2Gid(edge.get_neighbor()); if (dst_ids.find(gid) != dst_ids.end()) { diff --git a/analytical_engine/java/grape-giraph/src/main/java/com/alibaba/graphscope/parallel/mm/impl/GiraphMpiMessageManager.java b/analytical_engine/java/grape-giraph/src/main/java/com/alibaba/graphscope/parallel/mm/impl/GiraphMpiMessageManager.java index cb5fbd0476e1..b76c6cf1bc9b 100644 --- a/analytical_engine/java/grape-giraph/src/main/java/com/alibaba/graphscope/parallel/mm/impl/GiraphMpiMessageManager.java +++ b/analytical_engine/java/grape-giraph/src/main/java/com/alibaba/graphscope/parallel/mm/impl/GiraphMpiMessageManager.java @@ -113,11 +113,11 @@ public void receiveMessages() { /////////////////////////////////////////// bytesOfReceivedMsg += tmpVector.size(); } + tmpVector.delete(); logger.info( "Frag [{}] totally Received [{}] bytes from others starting deserialization", fragId, bytesOfReceivedMsg); - tmpVector.delete(); } /** @@ -256,5 +256,9 @@ public void postSuperstep() { } @Override - public void postApplication() {} + public void postApplication() { + for (int i = 0; i < fragNum; ++i) { + cacheOut[i].close(); + } + } } diff --git a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java index 543f32db60da..eb62dc095530 100644 --- a/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java +++ b/analytical_engine/java/grape-jdk/src/main/java/com/alibaba/graphscope/serialization/FFIByteVectorOutputStream.java @@ -38,6 +38,11 @@ public FFIByteVectorOutputStream(FFIByteVector vector) { offset = 0; } + @Override + public void close() { + vector.delete(); + } + public void resize(long size) { vector.resize(size); } @@ -412,9 +417,4 @@ public void write(int b) throws IOException { vector.setRawByte(offset, (byte) b); offset += 1; } - - @Override - public void close() throws IOException { - vector.delete(); - } } diff --git a/docs/index.rst b/docs/index.rst index b50a209bc91f..86d94a9228ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -74,6 +74,7 @@ and the vineyard store that offers efficient in-memory data transfers. interactive_engine/tinkerpop_eco interactive_engine/neo4j_eco interactive_engine/gopt + interactive_engine/graph_planner interactive_engine/benchmark_tool .. interactive_engine/guide_and_examples interactive_engine/design_of_gie diff --git a/docs/interactive_engine/graph_planner.md b/docs/interactive_engine/graph_planner.md new file mode 100644 index 000000000000..edef5ff9a16c --- /dev/null +++ b/docs/interactive_engine/graph_planner.md @@ -0,0 +1,234 @@ +# Graph Planner Interface by JNI and RESTful API + +`GraphPlanner` is the primary entry point for GOpt’s query optimization and physical plan generation. Originally, it was tightly integrated into the Frontend service, where it optimized Cypher queries received via the Bolt protocol and generated execution plans for various backend engines. + +To enhance its flexibility and ease of integration, `GraphPlanner` is now available as a standalone module, free from any dependencies on other Frontend modules. It supports both JNI and RESTful API interfaces, enabling lightweight and straightforward integration into diverse systems. Whether you are working on a native application or web-based services, `GraphPlanner` can seamlessly integrate into your architecture, providing efficient query optimization and physical plan generation across a wide range of use cases. + + +## JNI API + +### Interface Overview + +We provide a c++ wrapper implementation `GraphPlannerWrapper` for the JNI interface. Here is a brief explanation of the logical interface provided by the `c++` class. + +Constructor: + +```cpp +/** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) + */ +GraphPlannerWrapper(const std::string &java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml = "", + const std::string &graph_statistic_json = ""); +``` + +Method: + +```cpp +/** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format + * @return The physical plan in bytes and result schema in yaml. + */ +Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) +``` + +### Getting Started +Follow the steps below to get started with the Graph Planner interface for c++ invocation. + +#### Step 1: Build the Project + +Navigate to the project directory and build the package using Maven: +```bash +cd interactive_engine +mvn clean package -DskipTests -Pgraph-planner-jni +``` + +#### Step 2: Locate and Extract the Package + +After the build completes, a tarball named `graph-planner-jni.tar.gz` will be available in the `assembly/target` directory. Extract the contents of the tarball: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +#### Step 3: Run the Example Binary + +To demonstrate the usage of the JNI interface, an example binary `test_graph_planner` is provided. Use the following command to execute it: + +```bash +# bin/test_graph_planner +bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/gs_interactive_hiactor.yaml +``` + +The output consists of the physical plan (in byte format) and the result schema (in YAML format). The physical plan adheres to the specifications defined in the [protobuf](). + +Below is an example of a result schema: + +```yaml +schema: + name: default + description: default desc + mode: READ + extension: .so + library: libdefault.so + params: [] +returns: + - name: n + type: {primitive_type: DT_UNKNOWN} + - name: $f1 + type: {primitive_type: DT_SIGNED_INT64} +type: UNKNOWN +query: MATCH (n) RETURN n, COUNT(n); +``` + +The `returns` field defines the structure of the data returned by backend engines. Each nested entry in the returns field includes three components: +- the column name, which specifies the name of the result column; +- the entry’s ordinal position, which determines the column ID; +- the type, which enforces the data type constraint for the column. + +## Restful API + +We provide an alternative method to expose the interface as a RESTful API. Follow the steps below to access the interface via REST. + +### Getting Started + +#### Step 1: Build the Project + +To build the project, run the following command: +```bash +cd interactive_engine +# Use '-Dskip.native=true' to skip compiling C++ native code +mvn clean package -DskipTests -Pgraph-planner-jni -Dskip.native=true +``` + +#### Step 2: Locate and Extract the Package + +Once the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents as follows: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +#### Step 3: Start the Graph Planner RESTful Service + +To start the service, run the following command: + +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.restful.GraphPlannerService --spring.config.location=./conf/application.yaml +``` + +#### Step 4: Access the RESTful API by `Curl` + +To send a request to the RESTful API, use the following `curl` command: + +```bash +curl -X POST http://localhost:8080/api/compilePlan \ + -H "Content-Type: application/json" \ + -d "{ + \"configPath\": \"$configPath\", + \"query\": \"$query\", + \"schemaYaml\": \"$schemaYaml\", + \"statsJson\": \"$statsJson\" + }" +``` + +Replace `$configPath`, `$query`, `$schemaYaml`, and `$statsJson` with the appropriate values. + +The response will be in JSON format, similar to: + +```json +{ + "graphPlan": { + "physicalBytes": "", + "resultSchemaYaml": "" + } +} +``` + +The response contains two fields: +1. physicalBytes: A Base64-encoded string representing the physical plan bytes. +2. resultSchemaYaml: A string representing the YAML schema. + +You can decode these values into the required structures. + +#### Step 4: Access the RESTful API by `Java` Sdk + +Alternatively, if you are a java-side user, we provide a java sdk example to guide you how to access the restful API and decode the response : + +```java +public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); +} + +private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); +} + +// get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan +private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); +} + +// get result schema yaml from json +private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); +} +``` + +Run the java sdk example with the following command: +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/gs_interactive_hiactor.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json +``` \ No newline at end of file diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml new file mode 100644 index 000000000000..d77e506d51b6 --- /dev/null +++ b/interactive_engine/assembly/graph-planner-jni.xml @@ -0,0 +1,72 @@ + + graph-planner-jni + + tar.gz + + + + + ${project.parent.basedir}/executor/ir/target/release + + libir_core.* + + native + + + ${project.parent.basedir}/target/native/ + + libgraph_planner.* + + native + + + ${project.parent.basedir}/target/native/ + + test_graph_planner + + bin + 0755 + + + ${project.parent.basedir}/compiler/target/libs/ + libs + + + ${project.parent.basedir}/compiler/target/ + + compiler-0.0.1-SNAPSHOT.jar + + libs + + + ${project.parent.basedir}/compiler/conf + conf + + * + + + + ${project.parent.basedir}/../flex/interactive/examples/modern_graph/ + conf + + graph.yaml + + + + ${project.parent.basedir}/compiler/src/test/resources/statistics/ + conf + + modern_statistics.json + + + + ${project.parent.basedir}/compiler/src/test/resources/config/ + conf + + gs_interactive_hiactor.yaml + + + + diff --git a/interactive_engine/assembly/pom.xml b/interactive_engine/assembly/pom.xml index 39b0aa02ea32..a08a9a013d24 100644 --- a/interactive_engine/assembly/pom.xml +++ b/interactive_engine/assembly/pom.xml @@ -71,5 +71,22 @@ + + graph-planner-jni + + + + org.apache.maven.plugins + maven-assembly-plugin + + graph-planner-jni + + graph-planner-jni.xml + + + + + + diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml new file mode 100644 index 000000000000..0ffbb765fb46 --- /dev/null +++ b/interactive_engine/compiler/build_native.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/interactive_engine/compiler/conf/application.yaml b/interactive_engine/compiler/conf/application.yaml new file mode 100644 index 000000000000..d5e9d11a3746 --- /dev/null +++ b/interactive_engine/compiler/conf/application.yaml @@ -0,0 +1,22 @@ +# +# /* +# * Copyright 2020 Alibaba Group Holding Limited. +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +server: + port: 8080 +app: + name: GraphPlannerService \ No newline at end of file diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index 76f014bb17f3..6392385974b2 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -171,6 +171,10 @@ ch.qos.logback logback-classic + + org.springframework.boot + spring-boot-starter-web + @@ -294,6 +298,7 @@ build-helper-maven-plugin + generate-sources generate-sources add-source @@ -399,6 +404,7 @@ build_ir_core.sh + ${skip.ir.core} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index ec35991327ad..9669d119b5c2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -37,6 +37,9 @@ public class YamlConfigs extends Configs { "graph.planner.is.on", (Configs configs) -> configs.get("compiler.planner.is_on")) .put("graph.planner.opt", (Configs configs) -> configs.get("compiler.planner.opt")) + .put( + "graph.planner.cbo.glogue.size", + (Configs configs) -> configs.get("compiler.planner.cbo.glogue.size")) .put( "graph.planner.rules", (Configs configs) -> { @@ -65,6 +68,15 @@ public class YamlConfigs extends Configs { } return configs.get("compiler.meta.reader.statistics.uri"); }) + .put( + "graph.functions", + (Configs configs) -> { + String functions = System.getProperty("graph.functions"); + if (functions != null) { + return functions; + } + return configs.get("compiler.meta.reader.functions.uri"); + }) .put( "graph.meta.schema.fetch.interval.ms", (Configs configs) -> configs.get("compiler.meta.reader.schema.interval")) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java index 21374d84b07d..863bc6230b8a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/function/GraphFunctions.java @@ -27,6 +27,8 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; import java.io.ByteArrayInputStream; @@ -38,6 +40,7 @@ import java.util.Map; public class GraphFunctions { + private static final Logger logger = LoggerFactory.getLogger(GraphFunctions.class); public static final String FUNCTION_PREFIX = "gs.function."; private final Map functionMetaMap; @@ -85,7 +88,7 @@ private void registerConfigFunctions(Configs configs) { } } } catch (Exception e) { - throw new RuntimeException(e); + logger.warn("fail to load graph functions from configuration", e); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index c835bb613de7..2963ecb0ee3e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -174,16 +174,18 @@ public int hashCode() { } public static class Serializer { - public static void perform(StoredProcedureMeta meta, OutputStream outputStream) + public static void perform( + StoredProcedureMeta meta, OutputStream outputStream, boolean throwsOnFail) throws IOException { Yaml yaml = new Yaml(); - String mapStr = yaml.dump(createProduceMetaMap(meta)); + String mapStr = yaml.dump(createProduceMetaMap(meta, throwsOnFail)); outputStream.write(mapStr.getBytes(StandardCharsets.UTF_8)); } - private static Map createProduceMetaMap(StoredProcedureMeta meta) { + private static Map createProduceMetaMap( + StoredProcedureMeta meta, boolean throwsOnFail) { IrDataTypeConvertor typeConvertor = - new IrDataTypeConvertor.Flex(typeFactory); + new IrDataTypeConvertor.Flex(typeFactory, throwsOnFail); return ImmutableMap.of( Config.NAME.getKey(), meta.name, @@ -228,7 +230,7 @@ private static Map createProduceMetaMap(StoredProcedureMeta meta public static class Deserializer { public static StoredProcedureMeta perform(InputStream inputStream) throws IOException { IrDataTypeConvertor typeConvertor = - new IrDataTypeConvertor.Flex(typeFactory); + new IrDataTypeConvertor.Flex(typeFactory, true); Yaml yaml = new Yaml(); Map config = yaml.load(inputStream); return new StoredProcedureMeta( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java index 9cd921e6c975..fcaffd071382 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java @@ -186,9 +186,11 @@ public DataType convert(RelDataType dataFrom) { class Flex implements IrDataTypeConvertor { private final RelDataTypeFactory typeFactory; + private final boolean throwsOnFail; - public Flex(RelDataTypeFactory typeFactory) { + public Flex(RelDataTypeFactory typeFactory, boolean throwsOnFail) { this.typeFactory = typeFactory; + this.throwsOnFail = throwsOnFail; } @Override @@ -312,8 +314,11 @@ public RelDataType convert(GSDataTypeDesc from) { // decimal type with precision and scale return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale); } - throw new UnsupportedOperationException( - "convert GSDataTypeDesc [" + from + "] to RelDataType is unsupported yet"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "convert GSDataTypeDesc [" + from + "] to RelDataType is unsupported yet"); + } + return typeFactory.createSqlType(SqlTypeName.ANY); } @Override @@ -326,6 +331,9 @@ public GSDataTypeDesc convert(RelDataType from) { SqlTypeName typeName = from.getSqlTypeName(); Map yamlDesc; switch (typeName) { + case ANY: + yamlDesc = ImmutableMap.of("primitive_type", "DT_ANY"); + break; case NULL: yamlDesc = ImmutableMap.of("primitive_type", "DT_NULL"); break; @@ -420,10 +428,13 @@ public GSDataTypeDesc convert(RelDataType from) { "scale", from.getScale())); break; default: - throw new UnsupportedOperationException( - "convert RelDataType [" - + from - + "] to GSDataTypeDesc is unsupported yet"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "convert RelDataType [" + + from + + "] to GSDataTypeDesc is unsupported yet"); + } + yamlDesc = ImmutableMap.of("primitive_type", "DT_ANY"); } return new GSDataTypeDesc(yamlDesc); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java index 0e8d1a10da72..ffb1b9d511be 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java @@ -53,7 +53,7 @@ public static final GraphSchema buildSchemaFromYaml( Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); IrDataTypeConvertor typeConvertor = - new IrDataTypeConvertor.Flex(typeFactory); + new IrDataTypeConvertor.Flex(typeFactory, true); buildGraphElementFromYaml( (List) Objects.requireNonNull( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index e95e00a5930a..317cbfcadb4d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -275,7 +275,8 @@ public static final OuterExpression.ExprOpr protoOperator(SqlOperator operator) public static final com.alibaba.graphscope.proto.type.Common.DataType protoBasicDataType( RelDataType basicType) { try { - IrDataTypeConvertor convertor = new IrDataTypeConvertor.Flex(null); + IrDataTypeConvertor convertor = + new IrDataTypeConvertor.Flex(null, true); return convertor.convert(basicType).getProtoDesc(); } catch (Exception e) { throw new RuntimeException(e); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 948ec068e299..090d8f3d3a76 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -53,9 +53,7 @@ import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -227,16 +225,34 @@ private static Configs createExtraConfigs(@Nullable String extraYamlFile) throws return new Configs(keyValueMap); } - private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) - throws IOException { - URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); - if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - } else if (schemaUri.getScheme().equals("http")) { - return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); - } - throw new IllegalArgumentException( - "unknown graph meta reader mode: " + schemaUri.getScheme()); + public interface IrMetaFetcherFactory { + IrMetaFetcher create(Configs configs, IrMetaTracker tracker) throws IOException; + + IrMetaFetcherFactory DEFAULT = + (configs, tracker) -> { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + }; + } + + public static Summary generatePlan( + String configPath, String queryString, IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + Configs configs = Configs.Factory.create(configPath); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + PlannerInstance instance = planner.instance(queryString, metaFetcher.fetch().get()); + Summary summary = instance.plan(); + return summary; } public static void main(String[] args) throws Exception { @@ -250,18 +266,14 @@ public static void main(String[] args) throws Exception { + " '' ''" + " 'optional '"); } - Configs configs = Configs.Factory.create(args[0]); - GraphRelOptimizer optimizer = - new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); - IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); + String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); - GraphPlanner planner = - new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); - PlannerInstance instance = planner.instance(query, metaFetcher.fetch().get()); - Summary summary = instance.plan(); + + Summary summary = generatePlan(args[0], query, IrMetaFetcherFactory.DEFAULT); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; FileUtils.writeByteArrayToFile(new File(args[2]), physicalPlan.getContent()); + // write stored procedure meta to file LogicalPlan logicalPlan = summary.getLogicalPlan(); Configs extraConfigs = createExtraConfigs(args.length > 4 ? args[4] : null); @@ -271,6 +283,6 @@ public static void main(String[] args) throws Exception { query, logicalPlan.getOutputType(), logicalPlan.getDynamicParams()); - StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3])); + StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3]), true); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java new file mode 100644 index 000000000000..69820fad8964 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import java.io.Serializable; + +public class GraphPlan implements Serializable { + private final byte[] physicalBytes; + public String resultSchemaYaml; + + public GraphPlan(byte[] physicalBytes, String resultSchemaYaml) { + this.physicalBytes = physicalBytes; + this.resultSchemaYaml = resultSchemaYaml; + } + + public byte[] getPhysicalBytes() { + return physicalBytes; + } + + public String getResultSchemaYaml() { + return resultSchemaYaml; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java new file mode 100644 index 000000000000..ab5947a8095f --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; +import com.alibaba.graphscope.common.ir.planner.PlannerGroupManager; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlanFactory; + +public class GraphPlanerInstance { + private final GraphPlanner planner; + private final IrMeta meta; + + private static GraphPlanerInstance instance; + + public GraphPlanerInstance(GraphPlanner planner, IrMeta meta) { + this.planner = planner; + this.meta = meta; + } + + public static synchronized GraphPlanerInstance getInstance( + String configPath, GraphPlanner.IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + if (instance == null) { + Configs configs = Configs.Factory.create(configPath); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = + metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + instance = new GraphPlanerInstance(planner, metaFetcher.fetch().get()); + } + return instance; + } + + public GraphPlanner getPlanner() { + return planner; + } + + public IrMeta getMeta() { + return meta; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java new file mode 100644 index 000000000000..c5c753904a29 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java @@ -0,0 +1,125 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.GraphId; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphStatistics; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaInputStream; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaSpec; +import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; +import com.google.common.collect.ImmutableMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class PlanUtils { + private static final Logger logger = LoggerFactory.getLogger(PlanUtils.class); + /** + * Provide a java-side implementation to compile the query in string to a physical plan + * @param configPath + * @param query + * + * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, + * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, + * resultSchemaYaml defines the result specification of the query in yaml format + * @throws Exception + */ + public static GraphPlan compilePlan( + String configPath, String query, String schemaYaml, String statsJson) throws Exception { + long startTime = System.currentTimeMillis(); + GraphPlanerInstance instance = + GraphPlanerInstance.getInstance( + configPath, + (Configs configs, IrMetaTracker tracker) -> + new StaticIrMetaFetcher( + new StringMetaReader(schemaYaml, statsJson, configs), + tracker)); + GraphPlanner.PlannerInstance plannerInstance = + instance.getPlanner().instance(query, instance.getMeta()); + GraphPlanner.Summary summary = plannerInstance.plan(); + LogicalPlan logicalPlan = summary.getLogicalPlan(); + PhysicalPlan physicalPlan = summary.getPhysicalPlan(); + StoredProcedureMeta procedureMeta = + new StoredProcedureMeta( + new Configs(ImmutableMap.of()), + query, + logicalPlan.getOutputType(), + logicalPlan.getDynamicParams()); + ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); + StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); + long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("compile plan cost: {} ms", elapsedTime); + return new GraphPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); + } + + static class StringMetaReader implements IrMetaReader { + private final String schemaYaml; + private final String statsJson; + private final Configs configs; + + public StringMetaReader(String schemaYaml, String statsJson, Configs configs) { + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + this.configs = configs; + } + + @Override + public IrMeta readMeta() throws IOException { + IrGraphSchema graphSchema = + new IrGraphSchema( + configs, + new SchemaInputStream( + new ByteArrayInputStream( + schemaYaml.getBytes(StandardCharsets.UTF_8)), + SchemaSpec.Type.FLEX_IN_YAML)); + return new IrMeta( + graphSchema, + new GraphStoredProcedures( + new ByteArrayInputStream(schemaYaml.getBytes(StandardCharsets.UTF_8)), + this)); + } + + @Override + public GraphStatistics readStats(GraphId graphId) throws IOException { + return new IrGraphStatistics( + new ByteArrayInputStream(statsJson.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public boolean syncStatsEnabled(GraphId graphId) throws IOException { + return false; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java new file mode 100644 index 000000000000..1dc7274ef9f4 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java @@ -0,0 +1,91 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.examples; + +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class TestGraphPlanner { + public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); + } + + private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); + } + + // get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan + private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); + } + + // get result schema yaml from json + private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java new file mode 100644 index 000000000000..eb82798fba87 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java @@ -0,0 +1,69 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import java.io.Serializable; + +public class GraphPlanRequest implements Serializable { + private String configPath; + private String query; + private String schemaYaml; + private String statsJson; + + public GraphPlanRequest() {} + + public GraphPlanRequest(String configPath, String query, String schemaYaml, String statsJson) { + this.configPath = configPath; + this.query = query; + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + } + + public String getConfigPath() { + return configPath; + } + + public String getQuery() { + return query; + } + + public String getSchemaYaml() { + return schemaYaml; + } + + public String getStatsJson() { + return statsJson; + } + + public void setConfigPath(String configPath) { + this.configPath = configPath; + } + + public void setQuery(String query) { + this.query = query; + } + + public void setSchemaYaml(String schemaYaml) { + this.schemaYaml = schemaYaml; + } + + public void setStatsJson(String statsJson) { + this.statsJson = statsJson; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java new file mode 100644 index 000000000000..aad9d3d01ba1 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; + +import java.io.Serializable; +import java.util.Objects; + +public class GraphPlanResponse implements Serializable { + private GraphPlan graphPlan; + private String errorMessage; + + public GraphPlanResponse() {} + + public GraphPlanResponse(GraphPlan graphPlan) { + this.graphPlan = Objects.requireNonNull(graphPlan); + this.errorMessage = null; + } + + public GraphPlanResponse(String errorMessage) { + this.graphPlan = null; + this.errorMessage = Objects.requireNonNull(errorMessage); + } + + public GraphPlan getGraphPlan() { + return graphPlan; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setGraphPlan(GraphPlan graphPlan) { + this.graphPlan = graphPlan; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java new file mode 100644 index 000000000000..4cacf9e893c5 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java @@ -0,0 +1,54 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; +import com.alibaba.graphscope.sdk.PlanUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +@RequestMapping("/api") +public class GraphPlannerController { + private final Logger logger = LoggerFactory.getLogger(GraphPlannerController.class); + + @PostMapping("/compilePlan") + public ResponseEntity compilePlan(@RequestBody GraphPlanRequest request) { + try { + GraphPlan plan = + PlanUtils.compilePlan( + request.getConfigPath(), + request.getQuery(), + request.getSchemaYaml(), + request.getStatsJson()); + return ResponseEntity.ok(new GraphPlanResponse(plan)); + } catch (Exception e) { + logger.error("Failed to compile plan", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new GraphPlanResponse(e.getMessage())); + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java new file mode 100644 index 000000000000..7e2b9745d906 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java @@ -0,0 +1,29 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class GraphPlannerService { + public static void main(String[] args) { + SpringApplication.run(GraphPlannerService.class, args); + } +} diff --git a/interactive_engine/compiler/src/main/native/CMakeLists.txt b/interactive_engine/compiler/src/main/native/CMakeLists.txt new file mode 100644 index 000000000000..d8f0cefd8484 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/CMakeLists.txt @@ -0,0 +1,75 @@ +# Copyright 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.1) +project(COMPILER-JNI + LANGUAGES CXX + VERSION 0.0.1) + +option(BUILD_TEST "Whether to build test" ON) + +set(JAVA_AWT_INCLUDE_PATH NotNeeded) +set(CMAKE_CXX_STANDARD 17) +find_package(JNI QUIET) +if (JNI_FOUND) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH2}) +else() + message(FATAL_ERROR "JNI not found") +endif() + +find_package(Protobuf REQUIRED) +include_directories(${Protobuf_INCLUDE_DIRS}) + +set(GIE_COMPILER_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../executor/ir/proto/) +# proto files of gie compiler +set(CODE_GEN_PROTOBUF_FILES + ${GIE_COMPILER_PROTO_DIR}/algebra.proto + ${GIE_COMPILER_PROTO_DIR}/common.proto + ${GIE_COMPILER_PROTO_DIR}/expr.proto + ${GIE_COMPILER_PROTO_DIR}/physical.proto + ${GIE_COMPILER_PROTO_DIR}/results.proto + ${GIE_COMPILER_PROTO_DIR}/schema.proto + ${GIE_COMPILER_PROTO_DIR}/type.proto + ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto + ${GIE_COMPILER_PROTO_DIR}/basic_type.proto +) + +#create directory first +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie) + +# proto gen for gie proto +protobuf_generate(APPEND_PATH + TARGET ${LOCAL_EXE_NAME} + LANGUAGE cpp + OUT_VAR PROTO_SRCS_GIE + PROTOS ${CODE_GEN_PROTOBUF_FILES} + IMPORT_DIRS ${GIE_COMPILER_PROTO_DIR} + PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie +) + +add_library(ir_proto SHARED ${PROTO_SRCS_GIE}) + +target_link_libraries(ir_proto ${Protobuf_LIBRARIES}) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +file(GLOB_RECURSE PLANNER_SRC_FILES "graph_planner.cc") +add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) +target_link_libraries(graph_planner PUBLIC ir_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES} ${Protobuf_LIBRARIES}) +target_include_directories(graph_planner PUBLIC $) + +if (BUILD_TEST) + add_executable(test_graph_planner test/test.cc) + target_link_libraries(test_graph_planner PRIVATE graph_planner) +endif() diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc new file mode 100644 index 000000000000..d5dbbba57973 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.cc @@ -0,0 +1,464 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include +#include +#include +#include +#include +#include + +#include "graph_planner.h" +#include + +namespace gs +{ +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + + static JavaVM *_jvm = NULL; + + JavaVM *CreateJavaVM(const std::string &jvm_options) + { + const char *p, *q; + const char *jvm_opts; + if (jvm_options.empty()) + { + jvm_opts = getenv("FLEX_JVM_OPTS"); + } + else + { + jvm_opts = jvm_options.c_str(); + } + if (jvm_opts == NULL) + { + std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; + return NULL; + } + std::cout << "Jvm opts str: " << jvm_opts << std::endl; + + if (*jvm_opts == '\0') + return NULL; + + int num_of_opts = 1; + for (const char *p = jvm_opts; *p; p++) + { + if (*p == ' ') + num_of_opts++; + } + + if (num_of_opts == 0) + return NULL; + + JavaVM *jvm = NULL; + JNIEnv *env = NULL; + int i = 0; + int status = 1; + JavaVMInitArgs vm_args; + + JavaVMOption *options = new JavaVMOption[num_of_opts]; + memset(options, 0, sizeof(JavaVMOption) * num_of_opts); + + for (p = q = jvm_opts;; p++) + { + if (*p == ' ' || *p == '\0') + { + if (q >= p) + { + goto ret; + } + char *opt = new char[p - q + 1]; + memcpy(opt, q, p - q); + opt[p - q] = '\0'; + options[i++].optionString = opt; + q = p + 1; // assume opts are separated by single space + if (*p == '\0') + break; + } + } + + memset(&vm_args, 0, sizeof(vm_args)); + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = num_of_opts; + vm_args.options = options; + + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + if (status == JNI_OK) + { + std::cout << "Create java virtual machine successfully." << std::endl; + } + else if (status == JNI_EEXIST) + { + std::cout << "JNI evn already exists." << std::endl; + } + else + { + std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")" << std::endl; + } + + ret: + for (int i = 0; i < num_of_opts; i++) + { + delete[] options[i].optionString; + } + delete[] options; + return jvm; + } + + // One process can only create jvm for once. + JavaVM *GetJavaVM(const std::string jvm_options = "") + { + if (_jvm == NULL) + { + // Try to find whether there exists one javaVM + jsize nVMs; + JNI_GetCreatedJavaVMs(NULL, 0, + &nVMs); // 1. just get the required array length + std::cout << "Found " << nVMs << " VMs existing in this process." + << std::endl; + JavaVM **buffer = new JavaVM *[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + for (auto i = 0; i < nVMs; ++i) + { + if (buffer[i] != NULL) + { + _jvm = buffer[i]; + std::cout << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm) << std::endl; + return _jvm; + } + } + _jvm = CreateJavaVM(jvm_options); + std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; + } + return _jvm; + } + + JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + + JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) + { + if (!GetJavaVM(jvm_options)) + { + return; + } + int status = + GetJavaVM(jvm_options) + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + if (status != JNI_OK) + { + std::cerr << "Error attach current thread: " << status << std::endl; + } + } + + JNIEnvMark::~JNIEnvMark() + { + if (_env) + { + GetJavaVM()->DetachCurrentThread(); + } + } + + JNIEnv *JNIEnvMark::env() { return _env; } + + } // namespace jni + +#endif + + std::vector list_files(const std::string &path) + { + // list all files in the directory + std::vector files; + for (const auto &entry : std::filesystem::directory_iterator(path)) + { + files.push_back(entry.path().string()); + } + return files; + } + + void iterate_over_director(const std::string &dir_or_path, std::vector &output_paths) + { + if (dir_or_path.empty()) + { + return; + } + if (std::filesystem::is_directory(dir_or_path)) + { + auto files = list_files(dir_or_path); + output_paths.insert(output_paths.end(), files.begin(), files.end()); + } + else + { + output_paths.push_back(dir_or_path); + } + } + + std::string GraphPlannerWrapper::expand_directory(const std::string &path) + { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + iterate_over_director(sub_path, paths); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + iterate_over_director(sub_path, paths); + std::stringstream ss; + for (const auto &p : paths) + { + ss << p << ":"; + } + return ss.str(); + } + +#if (GRAPH_PLANNER_JNI_INVOKER) + + std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { + auto expanded_java_path = expand_directory(java_path); + std::cout << "Expanded java path: " << expanded_java_path << std::endl; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + // jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + // if (!graph_statistic_json.empty()) + // { + // jvm_options += " -Dgraph.statistics=" + graph_statistic_json; + // } + return jvm_options; + } + + Plan compilePlanJNI(jclass graph_planner_clz_, + jmethodID graph_planner_method_id_, JNIEnv *env, + const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + nullptr); + Plan plan; + if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) + { + std::cerr << "Invalid GraphPlannerWrapper." << std::endl; + return plan; + } + jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + jstring param3 = env->NewStringUTF(graph_schema_yaml.c_str()); + jstring param4 = env->NewStringUTF(graph_statistic_json.c_str()); + + // invoke jvm static function to get results as Object[] + jobject jni_plan = (jobject)env->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2, param3, param4); + + if (env->ExceptionCheck()) + { + env->ExceptionDescribe(); + env->ExceptionClear(); + std::cerr << "Error in calling GraphPlanner." << std::endl; + return plan; + } + + jmethodID method1 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getPhysicalBytes", "()[B"); + jmethodID method2 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getResultSchemaYaml", "()Ljava/lang/String;"); + + // 0-th object is the physical plan in byte array + jbyteArray res1 = (jbyteArray)env->CallObjectMethod(jni_plan, method1); + // 1-th object is the result schema in yaml format + jstring res2 = (jstring)env->CallObjectMethod(jni_plan, method2); + + if (res1 == NULL || res2 == NULL) + { + std::cerr << "Fail to generate plan." << std::endl; + return plan; + } + jbyte *str = env->GetByteArrayElements(res1, NULL); + jsize len = env->GetArrayLength(res1); + std::cout << "Physical plan size: " << len; + + plan.physical_plan.ParseFromArray(str, len); + plan.result_schema = env->GetStringUTFChars(res2, NULL); + + env->ReleaseByteArrayElements(res1, str, 0); + env->DeleteLocalRef(param1); + env->DeleteLocalRef(param2); + env->DeleteLocalRef(res1); + // remove new added jni objects + env->DeleteLocalRef(res2); + env->DeleteLocalRef(jni_plan); + + return plan; + } +#endif + +#if (!GRAPH_PLANNER_JNI_INVOKER) + + void write_query_to_pipe(const std::string &path, + const std::string &query_str) + { + std::cout << "write_query_to_pipe: " << path << std::endl; + + // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + int fd_to_java = open(path.c_str(), O_WRONLY); + if (fd_to_java < 0) + { + std::cerr << "Fail to open pipe: " << path << std::endl; + return; + } + std::cout << "open pipe done" << std::endl; + auto len = write(fd_to_java, query_str.c_str(), query_str.size()); + if (len != (int)query_str.size()) + { + std::cerr << "Fail to write query to pipe:" << len << std::endl; + return; + } + std::cout << "write_query_to_pipe done: " << len << std::endl; + close(fd_to_java); + } + + void write_query_to_file(const std::string &path, + const std::string &query_str) + { + std::ofstream query_file(path); + query_file << query_str; + query_file.close(); + } + + physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) + { + std::cout << "plan str size: " << plan_str.size() << std::endl; + physical::PhysicalPlan plan; + if (!plan.ParseFromString(plan_str)) + { + std::cerr << "Fail to parse physical plan." << std::endl; + return physical::PhysicalPlan(); + } + return plan; + } + + physical::PhysicalPlan + compilePlanSubprocess(const std::string &class_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json, + const std::string &compiler_config_path, + const std::string &cypher_query_string) + { + physical::PhysicalPlan physical_plan; + auto random_prefix = std::to_string( + std::chrono::system_clock::now().time_since_epoch().count()); + std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; + std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; + std::cout << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file << std::endl; + mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + + pid_t pid = fork(); + + if (pid == 0) + { + const char *const command_string_array[] = {"java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + GRAPH_PLANNER_FULL_NAME, + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; + execvp(command_string_array[0], + const_cast(command_string_array)); + } + else if (pid < 0) + { + std::cerr << "Error in fork." << std::endl; + } + else + { + write_query_to_pipe(dst_query_path, cypher_query_string); + + int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); + if (fd_from_java < 0) + { + std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; + return physical_plan; + } + std::vector stored_buffer; + char buffer[128]; + while (true) + { + ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); + if (bytesRead <= 0) + { + break; + } + stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); + } + physical_plan = readPhysicalPlan( + std::string(stored_buffer.begin(), stored_buffer.end())); + close(fd_from_java); + + int status; + waitpid(pid, &status, 0); + if (status != 0) + { + std::cerr << "Error in running command." << std::endl; + } + } + unlink(dst_query_path.c_str()); + unlink(dst_output_file.c_str()); + return physical_plan; + } +#endif + + /** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format + * @return The physical plan in bytes and result schema in yaml. + */ + Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, + jni_wrapper_.env(), compiler_config_path, + cypher_query_string, graph_schema_yaml, graph_statistic_json); +#else + return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, + graph_statistic_json_, compiler_config_path, + cypher_query_string); +#endif + } + +} // namespace gs diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h new file mode 100644 index 000000000000..40de6b3d0161 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.h @@ -0,0 +1,169 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#ifndef PLANNER_GRAPH_PLANNER_H_ +#define PLANNER_GRAPH_PLANNER_H_ + +#include +#include +#include +#include +#include +#include + +#include "flex/proto_generated_gie/physical.pb.h" + +#ifndef GRAPH_PLANNER_JNI_INVOKER +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess +#endif + +namespace gs +{ + + struct Plan + { + physical::PhysicalPlan physical_plan; + std::string result_schema; + }; + +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + struct JNIEnvMark + { + JNIEnv *_env; + + JNIEnvMark(); + JNIEnvMark(const std::string &jvm_options); + ~JNIEnvMark(); + JNIEnv *env(); + }; + + } // namespace jni +#endif + + class GraphPlannerWrapper + { + public: + static constexpr const char *kGraphPlannerClass = + "com/alibaba/graphscope/sdk/PlanUtils"; + static constexpr const char *GRAPH_PLANNER_FULL_NAME = + "com.alibaba.graphscope.sdk.PlanUtils"; + static constexpr const char *kGraphPlannerMethod = "compilePlan"; + static constexpr const char *kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/sdk/GraphPlan;"; + + /** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) + */ + GraphPlannerWrapper(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml = "", + const std::string &graph_statistic_json = "") +#if (GRAPH_PLANNER_JNI_INVOKER) + : jni_wrapper_(generate_jvm_options( + java_path, jna_path, graph_schema_yaml, graph_statistic_json)) + { + jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); + if (clz == NULL) + { + std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; + return; + } + graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); + jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( + graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); + if (j_method_id == NULL) + { + std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; + return; + } + graph_planner_method_id_ = j_method_id; + } +#else + : jna_path_("-Djna.library.path=" + jna_path), + graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), + graph_statistic_json_("-Dgraph.statistics=" + graph_statistic_json) + { + class_path_ = expand_directory(java_path); + } +#endif + + ~GraphPlannerWrapper() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + if (graph_planner_clz_ != NULL) + { + jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); + } +#endif + } + + inline bool is_valid() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; +#else + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. +#endif + } + + /** + * @brief Invoker GraphPlanner to generate a physical plan from a cypher + * query. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return physical plan in string. + */ + Plan CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); + + private: + std::string generate_jvm_options(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); + // physical::PhysicalPlan compilePlanJNI(const std::string& + // compiler_config_path, + // const std::string& + // cypher_query_string); + std::string expand_directory(const std::string &path); +#if (GRAPH_PLANNER_JNI_INVOKER) + // We need to list all files in the directory, if exists. + // The reason why we need to list all files in the directory is that + // java -Djava.class.path=dir/* (in jni, which we are using)will not load all + // jar files in the directory, While java -cp dir/* will load all jar files in + // the directory. + + gs::jni::JNIEnvMark jni_wrapper_; + jclass graph_planner_clz_; + jmethodID graph_planner_method_id_; +#else + std::string class_path_; + std::string jna_path_; + std::string graph_schema_yaml_; + std::string graph_statistic_json_; +#endif + }; +} // namespace gs + +#endif // PLANNER_GRAPH_PLANNER_H_ diff --git a/interactive_engine/compiler/src/main/native/test/test.cc b/interactive_engine/compiler/src/main/native/test/test.cc new file mode 100644 index 000000000000..1c58f11af6eb --- /dev/null +++ b/interactive_engine/compiler/src/main/native/test/test.cc @@ -0,0 +1,121 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include "graph_planner.h" + +#include +#include +#include + +#include + +std::string get_dir_name() +{ + // Get the directory of this source file + std::string current_dir = __FILE__; + size_t pos = current_dir.find_last_of("/"); + current_dir = current_dir.substr(0, pos); + return current_dir; +} + +void check_path_exits(const std::string &path) +{ + // split path by ':' + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + paths.push_back(sub_path); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + paths.push_back(sub_path); + + for (const auto &p : paths) + { + struct stat buffer; + if (stat(p.c_str(), &buffer) != 0) + { + std::cerr << "Path not exists: " << p << std::endl; + exit(1); + } + } + std::cout << "Path exists: " << path << std::endl; +} + +std::string read_string_from_file(const std::string &file_path) +{ + std::ifstream inputFile(file_path); // Open the file for reading + + if (!inputFile.is_open()) + { + std::cerr << "Error: Could not open the file " << file_path << std::endl; + exit(1); + } + // Use a stringstream to read the entire content of the file + std::ostringstream buffer; + buffer << inputFile.rdbuf(); // Read the file stream into the stringstream + + std::string fileContent = buffer.str(); // Get the string from the stringstream + inputFile.close(); // Close the file + + return fileContent; +} + +int main(int argc, char **argv) +{ + // Check if the correct number of arguments is provided + if (argc != 7) + { + std::cerr << "Usage: " << argv[0] + << " " << std::endl; + return 1; + } + + // auto start = std::chrono::high_resolution_clock::now(); + + std::string java_class_path = argv[1]; + std::string jna_class_path = argv[2]; + std::string graph_schema_path = argv[3]; + std::string graph_statistic_path = argv[4]; + + // check director or file exists + check_path_exits(java_class_path); + check_path_exits(jna_class_path); + check_path_exits(graph_schema_path); + check_path_exits(graph_statistic_path); + + gs::GraphPlannerWrapper graph_planner_wrapper( + java_class_path, jna_class_path); + + std::string schema_content = read_string_from_file(graph_schema_path); + std::string statistic_content = read_string_from_file(graph_statistic_path); + + std::string cypher_query_string = argv[5]; + std::string config_path = argv[6]; + auto plan = + graph_planner_wrapper.CompilePlan(config_path, cypher_query_string, schema_content, statistic_content); + + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start).count(); + + // std::cout << "total execution time: " << duration << " ms" << std::endl; + + std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; + std::cout << "schema: " << plan.result_schema << std::endl; + return 0; +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java index 7fc4f78a1f06..791f9c72e1f0 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java @@ -100,5 +100,8 @@ public void compiler_config_test() throws Exception { YamlConfigs configs = new YamlConfigs("config/gs_interactive_hiactor.yaml", FileLoadType.RESOURCES); Assert.assertEquals("UTF-8", FrontendConfig.CALCITE_DEFAULT_CHARSET.get(configs)); + Assert.assertEquals( + "./src/main/resources/conf/graph_functions.yaml", + GraphConfig.GRAPH_FUNCTIONS_URI.get(configs)); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java new file mode 100644 index 000000000000..4462c76dee87 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java @@ -0,0 +1,53 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import org.apache.commons.io.FileUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; + +public class JNICompilePlanTest { + private static String configPath; + private static String schemaYaml; + private static String statsJson; + + @BeforeClass + public static void before() throws Exception { + configPath = "src/test/resources/config/gs_interactive_hiactor.yaml"; + schemaYaml = + FileUtils.readFileToString( + new File("src/test/resources/schema/sls_schema.yaml"), + StandardCharsets.UTF_8); + statsJson = + FileUtils.readFileToString( + new File("src/test/resources/statistics/sls_statistics.json"), + StandardCharsets.UTF_8); + } + + @Test + public void path_expand_test() throws Exception { + String query = + "MATCH (src)-[e:test6*4..5]->(dest) WHERE src.__domain__ = 'xzz' RETURN" + + " src.__entity_id__ AS sId, dest.__entity_id__ AS dId;"; + PlanUtils.compilePlan(configPath, query, schemaYaml, statsJson); + } +} diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml index bc703b2d32ff..dc5a08bea71e 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml @@ -9,15 +9,23 @@ compute_engine: - localhost:10000 thread_num_per_worker: 1 compiler: + physical: + opt: + config: proto planner: is_on: true - opt: RBO + opt: CBO rules: + - FilterIntoJoinRule - FilterMatchRule + - ExtendIntersectRule + - ExpandGetVFusionRule meta: reader: schema: uri: ./target/test-classes/config/modern/graph.yaml + functions: + uri: ./src/main/resources/conf/graph_functions.yaml endpoint: default_listen_address: 0.0.0.0 # default localhost bolt_connector: # for cypher, there may be other connectors, such as bolt_connector, https_connector diff --git a/interactive_engine/compiler/src/test/resources/schema/sls_schema.yaml b/interactive_engine/compiler/src/test/resources/schema/sls_schema.yaml new file mode 100644 index 000000000000..3b04097495ad --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/schema/sls_schema.yaml @@ -0,0 +1,748 @@ +name: modern_graph # then must have a modern dir under ${data} directory +version: v0.1 +store_type: mutable_csr # v6d, groot, gart +description: xzz test t2 https://alidocs.dingtalk.com/i/nodes/G1DKw2zgV2KnvL4kFzx56elKJB5r9YAn +schema: + vertex_types: + - type_id: 0 + type_name: xzz@t5c + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 1 + type_name: xzz@t5a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 2 + type_name: xzz@t5b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 3 + type_name: xzz@t2a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 4 + type_name: xzz@t2b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 5 + type_name: xzz@t3b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 6 + type_name: xzz@t3a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 7 + type_name: xzz@t4b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 8 + type_name: xzz@t4c + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 9 + type_name: xzz@t6d + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 10 + type_name: xzz@t6e + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 11 + type_name: xzz@t3c + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 12 + type_name: xzz@t6c + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 13 + type_name: xzz@t6a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 14 + type_name: xzz@t6b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 15 + type_name: xzz@t5e + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 16 + type_name: xzz@t4e + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 17 + type_name: xzz@t4a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 18 + type_name: xzz@t5d + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 19 + type_name: xzz@t4d + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 20 + type_name: xzz@t1b + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + - type_id: 21 + type_name: xzz@t1a + description: A xzz test vertex type + properties: + - property_id: 0 + property_name: __domain__ + property_type: + string: + long_text: + - property_id: 1 + property_name: __entity_type__ + property_type: + string: + long_text: + - property_id: 2 + property_name: __entity_id__ + property_type: + string: + long_text: + - property_id: 3 + property_name: __label__ + property_type: + string: + long_text: + primary_keys: + - __entity_id__ + edge_types: + - type_id: 0 + type_name: test5 + description: A test5 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t5a + destination_vertex: xzz@t5b + relation: MANY_TO_MANY + - source_vertex: xzz@t5a + destination_vertex: xzz@t5c + relation: MANY_TO_MANY + - source_vertex: xzz@t5a + destination_vertex: xzz@t5d + relation: MANY_TO_MANY + - source_vertex: xzz@t5a + destination_vertex: xzz@t5e + relation: MANY_TO_MANY + - source_vertex: xzz@t5b + destination_vertex: xzz@t5b + relation: MANY_TO_MANY + - source_vertex: xzz@t5c + destination_vertex: xzz@t5c + relation: MANY_TO_MANY + - source_vertex: xzz@t5d + destination_vertex: xzz@t5d + relation: MANY_TO_MANY + - source_vertex: xzz@t5e + destination_vertex: xzz@t5e + relation: MANY_TO_MANY + - source_vertex: xzz@t5b + destination_vertex: xzz@t5a + relation: MANY_TO_MANY + - source_vertex: xzz@t5c + destination_vertex: xzz@t5a + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 1 + type_name: test2_1 + description: A test2_1 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t2a + destination_vertex: xzz@t2b + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 2 + type_name: test3 + description: A test3 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t3a + destination_vertex: xzz@t3a + relation: MANY_TO_MANY + - source_vertex: xzz@t3a + destination_vertex: xzz@t3b + relation: MANY_TO_MANY + - source_vertex: xzz@t3a + destination_vertex: xzz@t3c + relation: MANY_TO_MANY + - source_vertex: xzz@t3b + destination_vertex: xzz@t3a + relation: MANY_TO_MANY + - source_vertex: xzz@t3b + destination_vertex: xzz@t3b + relation: MANY_TO_MANY + - source_vertex: xzz@t3b + destination_vertex: xzz@t3c + relation: MANY_TO_MANY + - source_vertex: xzz@t3c + destination_vertex: xzz@t3a + relation: MANY_TO_MANY + - source_vertex: xzz@t3c + destination_vertex: xzz@t3b + relation: MANY_TO_MANY + - source_vertex: xzz@t3c + destination_vertex: xzz@t3c + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 3 + type_name: test4 + description: A test4 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t4a + destination_vertex: xzz@t4b + relation: MANY_TO_MANY + - source_vertex: xzz@t4b + destination_vertex: xzz@t4c + relation: MANY_TO_MANY + - source_vertex: xzz@t4c + destination_vertex: xzz@t4d + relation: MANY_TO_MANY + - source_vertex: xzz@t4d + destination_vertex: xzz@t4e + relation: MANY_TO_MANY + - source_vertex: xzz@t4e + destination_vertex: xzz@t4a + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 4 + type_name: test6 + description: A test6 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t6a + destination_vertex: xzz@t6b + relation: MANY_TO_MANY + - source_vertex: xzz@t6b + destination_vertex: xzz@t6c + relation: MANY_TO_MANY + - source_vertex: xzz@t6c + destination_vertex: xzz@t6d + relation: MANY_TO_MANY + - source_vertex: xzz@t6d + destination_vertex: xzz@t6e + relation: MANY_TO_MANY + - source_vertex: xzz@t6e + destination_vertex: xzz@t6e + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 5 + type_name: test2_2 + description: A test2_2 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t2a + destination_vertex: xzz@t2b + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: + - type_id: 6 + type_name: test1 + description: A test1 edge type + vertex_type_pair_relations: + - source_vertex: xzz@t1a + destination_vertex: xzz@t1b + relation: MANY_TO_MANY + - source_vertex: xzz@t1b + destination_vertex: xzz@t1a + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: __type__ + property_type: + string: + long_text: \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/statistics/sls_statistics.json b/interactive_engine/compiler/src/test/resources/statistics/sls_statistics.json new file mode 100644 index 000000000000..b4f240dccc74 --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/statistics/sls_statistics.json @@ -0,0 +1,325 @@ +{ + "total_vertex_count": 37, + "total_edge_count": 64, + "vertex_type_statistics": [ + { + "type_id": 0, + "type_name": "xzz@t5c", + "count": 2 + }, + { + "type_id": 1, + "type_name": "xzz@t5a", + "count": 1 + }, + { + "type_id": 2, + "type_name": "xzz@t5b", + "count": 2 + }, + { + "type_id": 3, + "type_name": "xzz@t2a", + "count": 2 + }, + { + "type_id": 4, + "type_name": "xzz@t2b", + "count": 2 + }, + { + "type_id": 5, + "type_name": "xzz@t3b", + "count": 2 + }, + { + "type_id": 6, + "type_name": "xzz@t3a", + "count": 2 + }, + { + "type_id": 7, + "type_name": "xzz@t4b", + "count": 1 + }, + { + "type_id": 8, + "type_name": "xzz@t4c", + "count": 1 + }, + { + "type_id": 9, + "type_name": "xzz@t6d", + "count": 2 + }, + { + "type_id": 10, + "type_name": "xzz@t6e", + "count": 2 + }, + { + "type_id": 11, + "type_name": "xzz@t3c", + "count": 2 + }, + { + "type_id": 12, + "type_name": "xzz@t6c", + "count": 1 + }, + { + "type_id": 13, + "type_name": "xzz@t6a", + "count": 2 + }, + { + "type_id": 14, + "type_name": "xzz@t6b", + "count": 4 + }, + { + "type_id": 15, + "type_name": "xzz@t5e", + "count": 2 + }, + { + "type_id": 16, + "type_name": "xzz@t4e", + "count": 1 + }, + { + "type_id": 17, + "type_name": "xzz@t4a", + "count": 1 + }, + { + "type_id": 18, + "type_name": "xzz@t5d", + "count": 2 + }, + { + "type_id": 19, + "type_name": "xzz@t4d", + "count": 1 + }, + { + "type_id": 20, + "type_name": "xzz@t1b", + "count": 1 + }, + { + "type_id": 21, + "type_name": "xzz@t1a", + "count": 1 + } + ], + "edge_type_statistics": [ + { + "type_id": 0, + "type_name": "test5", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t5a", + "destination_vertex": "xzz@t5b", + "count": 1 + }, + { + "source_vertex": "xzz@t5a", + "destination_vertex": "xzz@t5c", + "count": 1 + }, + { + "source_vertex": "xzz@t5a", + "destination_vertex": "xzz@t5d", + "count": 1 + }, + { + "source_vertex": "xzz@t5a", + "destination_vertex": "xzz@t5e", + "count": 1 + }, + { + "source_vertex": "xzz@t5b", + "destination_vertex": "xzz@t5b", + "count": 1 + }, + { + "source_vertex": "xzz@t5c", + "destination_vertex": "xzz@t5c", + "count": 1 + }, + { + "source_vertex": "xzz@t5d", + "destination_vertex": "xzz@t5d", + "count": 2 + }, + { + "source_vertex": "xzz@t5e", + "destination_vertex": "xzz@t5e", + "count": 1 + }, + { + "source_vertex": "xzz@t5b", + "destination_vertex": "xzz@t5a", + "count": 1 + }, + { + "source_vertex": "xzz@t5d", + "destination_vertex": "xzz@t5a", + "count": 1 + } + ] + }, + { + "type_id": 1, + "type_name": "test2_1", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t2a", + "destination_vertex": "xzz@t2b", + "count": 2 + } + ] + }, + { + "type_id": 2, + "type_name": "test3", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t3a", + "destination_vertex": "xzz@t3a", + "count": 2 + }, + { + "source_vertex": "xzz@t3a", + "destination_vertex": "xzz@t3b", + "count": 4 + }, + { + "source_vertex": "xzz@t3a", + "destination_vertex": "xzz@t3c", + "count": 4 + }, + { + "source_vertex": "xzz@t3b", + "destination_vertex": "xzz@t3a", + "count": 4 + }, + { + "source_vertex": "xzz@t3b", + "destination_vertex": "xzz@t3b", + "count": 2 + }, + { + "source_vertex": "xzz@t3b", + "destination_vertex": "xzz@t3c", + "count": 4 + }, + { + "source_vertex": "xzz@t3c", + "destination_vertex": "xzz@t3a", + "count": 4 + }, + { + "source_vertex": "xzz@t3c", + "destination_vertex": "xzz@t3b", + "count": 4 + }, + { + "source_vertex": "xzz@t3c", + "destination_vertex": "xzz@t3c", + "count": 2 + } + ] + }, + { + "type_id": 3, + "type_name": "test4", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t4a", + "destination_vertex": "xzz@t4b", + "count": 1 + }, + { + "source_vertex": "xzz@t4b", + "destination_vertex": "xzz@t4c", + "count": 1 + }, + { + "source_vertex": "xzz@t4c", + "destination_vertex": "xzz@t4d", + "count": 1 + }, + { + "source_vertex": "xzz@t4d", + "destination_vertex": "xzz@t4e", + "count": 1 + }, + { + "source_vertex": "xzz@t4e", + "destination_vertex": "xzz@t4a", + "count": 1 + } + ] + }, + { + "type_id": 4, + "type_name": "test6", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t6a", + "destination_vertex": "xzz@t6b", + "count": 4 + }, + { + "source_vertex": "xzz@t6b", + "destination_vertex": "xzz@t6c", + "count": 3 + }, + { + "source_vertex": "xzz@t6c", + "destination_vertex": "xzz@t6d", + "count": 2 + }, + { + "source_vertex": "xzz@t6d", + "destination_vertex": "xzz@t6e", + "count": 2 + }, + { + "source_vertex": "xzz@t6e", + "destination_vertex": "xzz@t6e", + "count": 1 + } + ] + }, + { + "type_id": 5, + "type_name": "test2_2", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t2a", + "destination_vertex": "xzz@t2b", + "count": 2 + } + ] + }, + { + "type_id": 6, + "type_name": "test1", + "vertex_type_pair_statistics": [ + { + "source_vertex": "xzz@t1a", + "destination_vertex": "xzz@t1b", + "count": 1 + }, + { + "source_vertex": "xzz@t1b", + "destination_vertex": "xzz@t1a", + "count": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/interactive_engine/executor/ir/core/src/plan/logical.rs b/interactive_engine/executor/ir/core/src/plan/logical.rs index f565d383f667..0d0290e7ec60 100644 --- a/interactive_engine/executor/ir/core/src/plan/logical.rs +++ b/interactive_engine/executor/ir/core/src/plan/logical.rs @@ -4944,4 +4944,22 @@ mod test { .unwrap() ); } + + #[test] + fn test_data_type_conversion() { + let schema = + Schema::from_json(std::fs::File::open("resource/modern_schema_pk.json").unwrap()).unwrap(); + for entity in schema.get_entities() { + let columns = &entity.columns; + for column in columns { + assert!(column.data_type.is_some()); + } + } + for relation in schema.get_relations() { + let columns = &relation.columns; + for column in columns { + assert!(&column.data_type.is_some()); + } + } + } } diff --git a/interactive_engine/executor/ir/core/src/plan/meta.rs b/interactive_engine/executor/ir/core/src/plan/meta.rs index 749ba82f1e9b..1930839252cb 100644 --- a/interactive_engine/executor/ir/core/src/plan/meta.rs +++ b/interactive_engine/executor/ir/core/src/plan/meta.rs @@ -231,6 +231,14 @@ impl Schema { (false, 0) } } + + pub(crate) fn get_entities(&self) -> &Vec { + self.entities.as_ref() + } + + pub(crate) fn get_relations(&self) -> &Vec { + self.relations.as_ref() + } } impl From for schema_pb::Schema { @@ -492,33 +500,36 @@ impl JsonIO for Schema { // } fn convert_data_type(data_type_int: i64) -> serde_json::Value { use serde_json::json; - match data_type_int { + let dt = match data_type_int { // Primitive types mapping - 0 => json!({ "primitive_type": "DT_BOOL" }), // BOOLEAN - 1 => json!({ "primitive_type": "DT_SIGNED_INT32" }), // INT32 - 2 => json!({ "primitive_type": "DT_SIGNED_INT64" }), // INT64 - 3 => json!({ "primitive_type": "DT_DOUBLE" }), // DOUBLE + 0 => json!({ "PrimitiveType": 5 }), // BOOLEAN + 1 => json!({ "PrimitiveType": 1 }), // INT32 + 2 => json!({ "PrimitiveType": 3 }), // INT64 + 3 => json!({ "PrimitiveType": 7 }), // DOUBLE // String type mapping - 4 => json!({ "string": { "long_text": {} } }), // STRING + 4 => json!({ "String": { "item": {"LongText": {}} } }), // STRING // Array types mapping - 6 => json!({ "array": { "component_type": { "primitive_type": "DT_SIGNED_INT32" } } }), // INT32_ARRAY - 7 => json!({ "array": { "component_type": { "primitive_type": "DT_SIGNED_INT64" } } }), // INT64_ARRAY - 8 => json!({ "array": { "component_type": { "primitive_type": "DT_DOUBLE" } } }), // DOUBLE_ARRAY - 9 => json!({ "array": { "component_type": { "string": { "long_text": {} } } } }), // STRING_ARRAY + 6 => json!({ "Array": { "component_type": { "item": {"PrimitiveType": 1 } }} , "max_length": 1024}), // INT32_ARRAY + 7 => json!({ "Array": { "component_type": { "item": {"PrimitiveType": 3 }} } , "max_length": 1024}), // INT64_ARRAY + 8 => json!({ "Array": { "component_type": { "item": {"PrimitiveType": 7 }} } , "max_length": 1024}), // DOUBLE_ARRAY + 9 => { + json!({ "Array": { "component_type": { "item": { "String": { "item": {"LongText": {}} } } } , "max_length": 1024} }) + } // STRING_ARRAY // None type mapping - 11 => json!({ "primitive_type": "DT_NULL" }), // NONE + 11 => json!({ "PrimitiveType": 8 }), // NONE // Temporal types mapping - 12 => json!({ "temporal": { "date32": {} } }), // DATE32 - 13 => json!({ "temporal": { "time32": {} } }), // TIME32 - 14 => json!({ "temporal": { "timestamp": {} } }), // TIMESTAMP + 12 => json!({ "Temporal": { "item": {"Date32": {} }} }), // DATE32 + 13 => json!({ "Temporal": {"item": { "Time32": {} }} }), // TIME32 + 14 => json!({ "Temporal": { "item": {"Timestamp": {} }} }), // TIMESTAMP // Other types handling (default to a NONE-like type if applicable) - _ => json!({ "primitive_type": "DT_ANY" }), // NONE or unsupported types - } + _ => json!({ "PrimitiveType": 0 }), // NONE or unsupported types + }; + json!({"item": dt}) } /// To define the options of required columns by the computing node of the query plan. diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index e5f55ca86aec..7463e68c0770 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -49,6 +49,48 @@ compiler + + graph-planner-jni + + false + true + + + false + + + common + executor/engine/pegasus/clients/java/client + compiler + assembly + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + + run + + compile + + + + + + + ${skip.native} + + + + false + + + + graphscope @@ -698,6 +740,11 @@ ${interactive.sdk.version} ${interactive.sdk.classifier} + + org.springframework.boot + spring-boot-starter-web + 2.5.12 + @@ -885,6 +932,11 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.0.0 + diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index 4acc372d2cb9..6682d5612643 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -903,7 +903,15 @@ install_interactive_dependencies() { fi # opentelemetry if [[ "${OS_PLATFORM}" != *"Darwin"* ]]; then - install_opentelemetry + # opentelemetry expect libprotoc >= 3.13.0, see https://github.com/open-telemetry/opentelemetry-cpp/discussions/2223 + proto_version=$(protoc --version | awk '{print $2}') + major_version=$(echo ${proto_version} | cut -d'.' -f1) + minor_version=$(echo ${proto_version} | cut -d'.' -f2) + if [[ ${major_version} -lt 3 ]] || [[ ${major_version} -eq 3 && ${minor_version} -lt 13 ]]; then + warning "OpenTelemetry requires protoc >= 3.13, current version is ${proto_version}, please upgrade it." + else + install_opentelemetry + fi fi } diff --git a/python/graphscope/tests/kubernetes/test_demo_script.py b/python/graphscope/tests/kubernetes/test_demo_script.py index e548ef9a469e..d5d6d74c7e7d 100644 --- a/python/graphscope/tests/kubernetes/test_demo_script.py +++ b/python/graphscope/tests/kubernetes/test_demo_script.py @@ -339,6 +339,7 @@ def test_vineyard_deployment_on_multiple_nodes( ) +@pytest.mark.skip(reason="compile too slow") def test_demo_distribute(gs_session_distributed, data_dir, modern_graph_data_dir): graph = load_ldbc(gs_session_distributed, data_dir)