diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 716c79ac2627..bd7c3ddb178e 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -317,6 +317,25 @@ jobs: cd ${GITHUB_WORKSPACE}/interactive_engine/ mvn clean install -Pexperimental -DskipTests -q + - name: Test physical plan generation + run: | + cd ${GITHUB_WORKSPACE}/interactive_engine/compiler + cat > /tmp/physical_plan_gen_config.yaml <> /tmp/physical_plan_gen_config.yaml + echo " meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml + mvn clean install -DskipTests -Pgraph-planner-jni + ./target/native/test_graph_planner + - name: Run End-to-End cypher adhoc ldbc query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/flex/tests/CMakeLists.txt b/flex/tests/CMakeLists.txt index e2bb952d7eb9..4c851c54b9a4 100644 --- a/flex/tests/CMakeLists.txt +++ b/flex/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_subdirectory(hqps) -add_subdirectory(rt_mutable_graph) +add_subdirectory(rt_mutable_graph) \ No newline at end of file diff --git a/interactive_engine/assembly/graph-planner-jni.xml b/interactive_engine/assembly/graph-planner-jni.xml new file mode 100644 index 000000000000..3413eed2a566 --- /dev/null +++ b/interactive_engine/assembly/graph-planner-jni.xml @@ -0,0 +1,72 @@ + + graph-planner-jni + + tar.gz + + + + + ${project.parent.basedir}/executor/ir/target/release + + libir_core.* + + native + + + ${project.parent.basedir}/target/native/ + + libgraph_planner.* + + native + + + ${project.parent.basedir}/target/native/ + + test_graph_planner + + bin + 0755 + + + ${project.parent.basedir}/compiler/target/libs/ + libs + + + ${project.parent.basedir}/compiler/target/ + + compiler-0.0.1-SNAPSHOT.jar + + libs + + + ${project.parent.basedir}/compiler/conf + conf + + * + + + + ${project.parent.basedir}/../flex/interactive/examples/modern_graph/ + conf + + graph.yaml + + + + ${project.parent.basedir}/compiler/src/test/resources/statistics/ + conf + + modern_statistics.json + + + + ${project.parent.basedir}/compiler/src/test/resources/config/ + conf + + interactive_config_test.yaml + + + + diff --git a/interactive_engine/assembly/pom.xml b/interactive_engine/assembly/pom.xml index 39b0aa02ea32..a08a9a013d24 100644 --- a/interactive_engine/assembly/pom.xml +++ b/interactive_engine/assembly/pom.xml @@ -71,5 +71,22 @@ + + graph-planner-jni + + + + org.apache.maven.plugins + maven-assembly-plugin + + graph-planner-jni + + graph-planner-jni.xml + + + + + + diff --git a/interactive_engine/compiler/build_ir_core.sh b/interactive_engine/compiler/build_ir_core.sh index 532e2f0a5f64..3a27f9725c6d 100755 --- a/interactive_engine/compiler/build_ir_core.sh +++ b/interactive_engine/compiler/build_ir_core.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -e -set -x - -if $(! command -v cargo &> /dev/null) -then - echo "cargo not exit, skip compile" -else - cd ../executor/ir/core - cargo build --release -fi +#set -e +#set -x +# +#if $(! command -v cargo &> /dev/null) +#then +# echo "cargo not exit, skip compile" +#else +# cd ../executor/ir/core +# cargo build --release +#fi diff --git a/interactive_engine/compiler/build_native.xml b/interactive_engine/compiler/build_native.xml new file mode 100644 index 000000000000..0ffbb765fb46 --- /dev/null +++ b/interactive_engine/compiler/build_native.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/interactive_engine/compiler/conf/application.yaml b/interactive_engine/compiler/conf/application.yaml new file mode 100644 index 000000000000..d5e9d11a3746 --- /dev/null +++ b/interactive_engine/compiler/conf/application.yaml @@ -0,0 +1,22 @@ +# +# /* +# * Copyright 2020 Alibaba Group Holding Limited. +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +server: + port: 8080 +app: + name: GraphPlannerService \ No newline at end of file diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 992f828a2b2e..5a13dc9d4f43 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json graph.store: exp graph.planner.is.on: true -graph.planner.opt: RBO +graph.planner.opt: CBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule # set statistics access uri -# graph.statistics: src/test/resources/statistics/modern_statistics.json +graph.statistics: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: @@ -60,7 +60,7 @@ calcite.default.charset: UTF-8 # gremlin.script.language.name: antlr_gremlin_traversal # the output plan format, can be ffi(default) or proto -# graph.physical.opt: ffi +graph.physical.opt: proto # set the max capacity of the result streaming buffer for each query # per.query.stream.buffer.max.capacity: 256 diff --git a/interactive_engine/compiler/pom.xml b/interactive_engine/compiler/pom.xml index 76f014bb17f3..0f88b8a48834 100644 --- a/interactive_engine/compiler/pom.xml +++ b/interactive_engine/compiler/pom.xml @@ -171,6 +171,10 @@ ch.qos.logback logback-classic + + org.springframework.boot + spring-boot-starter-web + @@ -294,6 +298,7 @@ build-helper-maven-plugin + generate-sources generate-sources add-source diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index ab11c9b198b2..9a28564c1812 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -210,8 +210,8 @@ private static GraphProperties getTestGraph(Configs configs) { case "rust-mcsr": testGraph = TestGraphFactory.RUST_MCSR; break; - case "cpp-mcsr": - logger.info("using cpp-mcsr as test graph"); + case "native-mcsr": + logger.info("using native-mcsr as test graph"); testGraph = TestGraphFactory.CPP_MCSR; break; default: diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index e55682a86cdb..438d1b22b211 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -37,6 +37,9 @@ public class YamlConfigs extends Configs { "graph.planner.is.on", (Configs configs) -> configs.get("compiler.planner.is_on")) .put("graph.planner.opt", (Configs configs) -> configs.get("compiler.planner.opt")) + .put( + "graph.planner.cbo.glogue.size", + (Configs configs) -> configs.get("compiler.planner.cbo.glogue.size")) .put( "graph.planner.rules", (Configs configs) -> { @@ -78,7 +81,7 @@ public class YamlConfigs extends Configs { if (configs.get("compute_engine.store.type") != null) { return configs.get("compute_engine.store.type"); } else { - return "cpp-mcsr"; + return "native-mcsr"; } }) .put( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java index b48cc07d81a1..2afd866ad484 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java @@ -51,7 +51,7 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throw try { return this.reader.readStats(meta.getGraphId()); } catch (Exception e) { - logger.warn("failed to read graph statistics, error is {}", e); + logger.warn("failed to read graph statistics, error is: " + e); return null; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index c0e00ddde078..d2d9c1e329ee 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -163,16 +163,19 @@ public int hashCode() { } public static class Serializer { - public static void perform(StoredProcedureMeta meta, OutputStream outputStream) + public static void perform( + StoredProcedureMeta meta, OutputStream outputStream, boolean throwsOnFail) throws IOException { Yaml yaml = new Yaml(); - String mapStr = yaml.dump(createProduceMetaMap(meta)); + String mapStr = yaml.dump(createProduceMetaMap(meta, throwsOnFail)); outputStream.write(mapStr.getBytes(StandardCharsets.UTF_8)); } - private static Map createProduceMetaMap(StoredProcedureMeta meta) { + private static Map createProduceMetaMap( + StoredProcedureMeta meta, boolean throwsOnFail) { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + GSDataTypeConvertor.Factory.create( + RelDataType.class, typeFactory, throwsOnFail); return ImmutableMap.of( Config.NAME.getKey(), meta.name, @@ -217,7 +220,7 @@ private static Map createProduceMetaMap(StoredProcedureMeta meta public static class Deserializer { public static StoredProcedureMeta perform(InputStream inputStream) throws IOException { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory, true); Yaml yaml = new Yaml(); Map config = yaml.load(inputStream); return new StoredProcedureMeta( @@ -346,7 +349,7 @@ public static class Config { // option configurations. public static final com.alibaba.graphscope.common.config.Config TYPE = com.alibaba.graphscope.common.config.Config.stringConfig( - "type", "UNKNOWN"); // cypher or cpp + "type", "UNKNOWN"); // cypher or native public static final com.alibaba.graphscope.common.config.Config QUERY = com.alibaba.graphscope.common.config.Config.stringConfig("query", "UNKNOWN"); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java index f12a70e8caf8..8eaa8898d032 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java @@ -38,7 +38,8 @@ public interface GSDataTypeConvertor { GSDataTypeDesc convert(T from); class Factory { - public static GSDataTypeConvertor create(Class tType, @Nullable Object config) { + public static GSDataTypeConvertor create( + Class tType, @Nullable Object config, boolean throwsOnFail) { if (tType.equals(DataType.class)) { return new GSDataTypeConvertor() { @Override @@ -58,20 +59,24 @@ public DataType convert(GSDataTypeDesc from) { case "DT_DOUBLE": return DataType.DOUBLE; default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } } } else if ((value = typeMap.get("string")) != null) { Map strType = (Map) value; if (strType.containsKey("long_text")) { return DataType.STRING; } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } } } else if ((value = typeMap.get("temporal")) != null) { Map temporalType = (Map) value; @@ -82,15 +87,22 @@ public DataType convert(GSDataTypeDesc from) { } else if (temporalType.containsKey("timestamp")) { return DataType.TIMESTAMP; } else { + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to DataType"); + } + } + } else { + if (throwsOnFail) { throw new UnsupportedOperationException( "can not convert GSDataTypeDesc [" + from + "] to DataType"); } - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); } + return DataType.UNKNOWN; } @Override @@ -121,20 +133,24 @@ public RelDataType convert(GSDataTypeDesc from) { case "DT_DOUBLE": return typeFactory.createSqlType(SqlTypeName.DOUBLE); default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("string")) != null) { Map strType = (Map) value; if (strType.containsKey("long_text")) { return typeFactory.createSqlType(SqlTypeName.CHAR); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("temporal")) != null) { Map temporalType = (Map) value; @@ -145,10 +161,12 @@ public RelDataType convert(GSDataTypeDesc from) { } else if (temporalType.containsKey("timestamp")) { return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } } else if ((value = typeMap.get("array")) != null) { Map arrayType = (Map) value; @@ -174,9 +192,14 @@ public RelDataType convert(GSDataTypeDesc from) { convert(new GSDataTypeDesc(keyType)), convert(new GSDataTypeDesc(valueType))); } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + + from + + "] to RelDataType"); + } } + return typeFactory.createUnknownType(); } @Override @@ -256,10 +279,14 @@ public GSDataTypeDesc convert(RelDataType from) { valueType)); break; default: - throw new UnsupportedOperationException( - "can not convert RelDataType [" - + from - + "] to GSDataTypeDesc"); + if (throwsOnFail) { + throw new UnsupportedOperationException( + "can not convert RelDataType [" + + from + + "] to GSDataTypeDesc"); + } else { + yamlDesc = ImmutableMap.of("primitive_type", "DT_UNKNOWN"); + } } return new GSDataTypeDesc(yamlDesc); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java index 0b59ccf7d216..fef0aad52841 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java @@ -52,7 +52,7 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(DataType.class, null); + GSDataTypeConvertor.Factory.create(DataType.class, null, true); builderGraphElementFromYaml( (List) Objects.requireNonNull( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 948ec068e299..bc4ef8184a88 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -53,9 +53,7 @@ import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -227,16 +225,34 @@ private static Configs createExtraConfigs(@Nullable String extraYamlFile) throws return new Configs(keyValueMap); } - private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) - throws IOException { - URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); - if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - } else if (schemaUri.getScheme().equals("http")) { - return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); - } - throw new IllegalArgumentException( - "unknown graph meta reader mode: " + schemaUri.getScheme()); + public interface IrMetaFetcherFactory { + IrMetaFetcher create(Configs configs, IrMetaTracker tracker) throws IOException; + + IrMetaFetcherFactory DEFAULT = + (configs, tracker) -> { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + }; + } + + public static Summary generatePlan( + String configPath, String queryString, IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + Configs configs = Configs.Factory.create(configPath); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + PlannerInstance instance = planner.instance(queryString, metaFetcher.fetch().get()); + Summary summary = instance.plan(); + return summary; } public static void main(String[] args) throws Exception { @@ -250,18 +266,23 @@ public static void main(String[] args) throws Exception { + " '' ''" + " 'optional '"); } - Configs configs = Configs.Factory.create(args[0]); - GraphRelOptimizer optimizer = - new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); - IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); - String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); - GraphPlanner planner = - new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); - PlannerInstance instance = planner.instance(query, metaFetcher.fetch().get()); - Summary summary = instance.plan(); + + BufferedReader reader = new BufferedReader(new FileReader(args[1])); + StringBuilder builder = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + builder.append(line); + } + String query = builder.toString(); + reader.close(); + + Summary summary = generatePlan(args[0], query, IrMetaFetcherFactory.DEFAULT); // write physical plan to file PhysicalPlan physicalPlan = summary.physicalPlan; - FileUtils.writeByteArrayToFile(new File(args[2]), physicalPlan.getContent()); + FileOutputStream fos = new FileOutputStream(args[2]); + fos.write(physicalPlan.getContent()); + fos.close(); + // write stored procedure meta to file LogicalPlan logicalPlan = summary.getLogicalPlan(); Configs extraConfigs = createExtraConfigs(args.length > 4 ? args[4] : null); @@ -271,6 +292,6 @@ public static void main(String[] args) throws Exception { query, logicalPlan.getOutputType(), logicalPlan.getDynamicParams()); - StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3])); + StoredProcedureMeta.Serializer.perform(procedureMeta, new FileOutputStream(args[3]), true); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java new file mode 100644 index 000000000000..69820fad8964 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlan.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import java.io.Serializable; + +public class GraphPlan implements Serializable { + private final byte[] physicalBytes; + public String resultSchemaYaml; + + public GraphPlan(byte[] physicalBytes, String resultSchemaYaml) { + this.physicalBytes = physicalBytes; + this.resultSchemaYaml = resultSchemaYaml; + } + + public byte[] getPhysicalBytes() { + return physicalBytes; + } + + public String getResultSchemaYaml() { + return resultSchemaYaml; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java new file mode 100644 index 000000000000..ab5947a8095f --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/GraphPlanerInstance.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; +import com.alibaba.graphscope.common.ir.planner.PlannerGroupManager; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlanFactory; + +public class GraphPlanerInstance { + private final GraphPlanner planner; + private final IrMeta meta; + + private static GraphPlanerInstance instance; + + public GraphPlanerInstance(GraphPlanner planner, IrMeta meta) { + this.planner = planner; + this.meta = meta; + } + + public static synchronized GraphPlanerInstance getInstance( + String configPath, GraphPlanner.IrMetaFetcherFactory metaFetcherFactory) + throws Exception { + if (instance == null) { + Configs configs = Configs.Factory.create(configPath); + GraphRelOptimizer optimizer = + new GraphRelOptimizer(configs, PlannerGroupManager.Static.class); + IrMetaFetcher metaFetcher = + metaFetcherFactory.create(configs, optimizer.getGlogueHolder()); + GraphPlanner planner = + new GraphPlanner(configs, new LogicalPlanFactory.Cypher(), optimizer); + instance = new GraphPlanerInstance(planner, metaFetcher.fetch().get()); + } + return instance; + } + + public GraphPlanner getPlanner() { + return planner; + } + + public IrMeta getMeta() { + return meta; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java new file mode 100644 index 000000000000..547cadcfc0c5 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/PlanUtils.java @@ -0,0 +1,121 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.GraphId; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphStatistics; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaInputStream; +import com.alibaba.graphscope.common.ir.meta.schema.SchemaSpec; +import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; +import com.google.common.collect.ImmutableMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class PlanUtils { + private static final Logger logger = LoggerFactory.getLogger(PlanUtils.class); + /** + * Provide a java-side implementation to compile the query in string to a physical plan + * @param configPath + * @param query + * + * @return JNIPlan has two fields: physicalBytes and resultSchemaYaml, + * physicalBytes can be decoded to {@code PhysicalPlan} in c++ side by standard PB serialization, + * resultSchemaYaml defines the result specification of the query in yaml format + * @throws Exception + */ + public static GraphPlan compilePlan( + String configPath, String query, String schemaYaml, String statsJson) throws Exception { + long startTime = System.currentTimeMillis(); + GraphPlanerInstance instance = + GraphPlanerInstance.getInstance( + configPath, + (Configs configs, IrMetaTracker tracker) -> + new StaticIrMetaFetcher( + new StringMetaReader(schemaYaml, statsJson), tracker)); + GraphPlanner.PlannerInstance plannerInstance = + instance.getPlanner().instance(query, instance.getMeta()); + GraphPlanner.Summary summary = plannerInstance.plan(); + LogicalPlan logicalPlan = summary.getLogicalPlan(); + PhysicalPlan physicalPlan = summary.getPhysicalPlan(); + StoredProcedureMeta procedureMeta = + new StoredProcedureMeta( + new Configs(ImmutableMap.of()), + query, + logicalPlan.getOutputType(), + logicalPlan.getDynamicParams()); + ByteArrayOutputStream metaStream = new ByteArrayOutputStream(); + StoredProcedureMeta.Serializer.perform(procedureMeta, metaStream, false); + long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("compile plan cost: {} ms", elapsedTime); + return new GraphPlan(physicalPlan.getContent(), new String(metaStream.toByteArray())); + } + + static class StringMetaReader implements IrMetaReader { + private final String schemaYaml; + private final String statsJson; + + public StringMetaReader(String schemaYaml, String statsJson) { + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + } + + @Override + public IrMeta readMeta() throws IOException { + IrGraphSchema graphSchema = + new IrGraphSchema( + new SchemaInputStream( + new ByteArrayInputStream( + schemaYaml.getBytes(StandardCharsets.UTF_8)), + SchemaSpec.Type.FLEX_IN_YAML)); + return new IrMeta( + graphSchema, + new GraphStoredProcedures( + new ByteArrayInputStream(schemaYaml.getBytes(StandardCharsets.UTF_8)), + this)); + } + + @Override + public GraphStatistics readStats(GraphId graphId) throws IOException { + return new IrGraphStatistics( + new ByteArrayInputStream(statsJson.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public boolean syncStatsEnabled(GraphId graphId) throws IOException { + return false; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java new file mode 100644 index 000000000000..1dc7274ef9f4 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/examples/TestGraphPlanner.java @@ -0,0 +1,91 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.examples; + +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class TestGraphPlanner { + public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); + } + + private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); + } + + // get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan + private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); + } + + // get result schema yaml from json + private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java new file mode 100644 index 000000000000..eb82798fba87 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanRequest.java @@ -0,0 +1,69 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import java.io.Serializable; + +public class GraphPlanRequest implements Serializable { + private String configPath; + private String query; + private String schemaYaml; + private String statsJson; + + public GraphPlanRequest() {} + + public GraphPlanRequest(String configPath, String query, String schemaYaml, String statsJson) { + this.configPath = configPath; + this.query = query; + this.schemaYaml = schemaYaml; + this.statsJson = statsJson; + } + + public String getConfigPath() { + return configPath; + } + + public String getQuery() { + return query; + } + + public String getSchemaYaml() { + return schemaYaml; + } + + public String getStatsJson() { + return statsJson; + } + + public void setConfigPath(String configPath) { + this.configPath = configPath; + } + + public void setQuery(String query) { + this.query = query; + } + + public void setSchemaYaml(String schemaYaml) { + this.schemaYaml = schemaYaml; + } + + public void setStatsJson(String statsJson) { + this.statsJson = statsJson; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java new file mode 100644 index 000000000000..aad9d3d01ba1 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlanResponse.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; + +import java.io.Serializable; +import java.util.Objects; + +public class GraphPlanResponse implements Serializable { + private GraphPlan graphPlan; + private String errorMessage; + + public GraphPlanResponse() {} + + public GraphPlanResponse(GraphPlan graphPlan) { + this.graphPlan = Objects.requireNonNull(graphPlan); + this.errorMessage = null; + } + + public GraphPlanResponse(String errorMessage) { + this.graphPlan = null; + this.errorMessage = Objects.requireNonNull(errorMessage); + } + + public GraphPlan getGraphPlan() { + return graphPlan; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setGraphPlan(GraphPlan graphPlan) { + this.graphPlan = graphPlan; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java new file mode 100644 index 000000000000..4cacf9e893c5 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerController.java @@ -0,0 +1,54 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import com.alibaba.graphscope.sdk.GraphPlan; +import com.alibaba.graphscope.sdk.PlanUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +@RequestMapping("/api") +public class GraphPlannerController { + private final Logger logger = LoggerFactory.getLogger(GraphPlannerController.class); + + @PostMapping("/compilePlan") + public ResponseEntity compilePlan(@RequestBody GraphPlanRequest request) { + try { + GraphPlan plan = + PlanUtils.compilePlan( + request.getConfigPath(), + request.getQuery(), + request.getSchemaYaml(), + request.getStatsJson()); + return ResponseEntity.ok(new GraphPlanResponse(plan)); + } catch (Exception e) { + logger.error("Failed to compile plan", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new GraphPlanResponse(e.getMessage())); + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java new file mode 100644 index 000000000000..7e2b9745d906 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/sdk/restful/GraphPlannerService.java @@ -0,0 +1,29 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk.restful; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class GraphPlannerService { + public static void main(String[] args) { + SpringApplication.run(GraphPlannerService.class, args); + } +} diff --git a/interactive_engine/compiler/src/main/native/CMakeLists.txt b/interactive_engine/compiler/src/main/native/CMakeLists.txt new file mode 100644 index 000000000000..e4157c2aa8ae --- /dev/null +++ b/interactive_engine/compiler/src/main/native/CMakeLists.txt @@ -0,0 +1,74 @@ +# Copyright 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.1) +project(COMPILER-JNI + LANGUAGES CXX + VERSION 0.0.1) + +option(BUILD_TEST "Whether to build test" ON) + +set(JAVA_AWT_INCLUDE_PATH NotNeeded) +set(CMAKE_CXX_STANDARD 17) +find_package(JNI QUIET) +if (JNI_FOUND) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH}) + include_directories(SYSTEM ${JAVA_INCLUDE_PATH2}) +else() + message(FATAL_ERROR "JNI not found") +endif() + +find_package(Protobuf REQUIRED) +include_directories(${Protobuf_INCLUDE_DIRS}) + +set(GIE_COMPILER_PROTO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../executor/ir/proto/) +# proto files of gie compiler +set(CODE_GEN_PROTOBUF_FILES + ${GIE_COMPILER_PROTO_DIR}/algebra.proto + ${GIE_COMPILER_PROTO_DIR}/common.proto + ${GIE_COMPILER_PROTO_DIR}/expr.proto + ${GIE_COMPILER_PROTO_DIR}/physical.proto + ${GIE_COMPILER_PROTO_DIR}/results.proto + ${GIE_COMPILER_PROTO_DIR}/schema.proto + ${GIE_COMPILER_PROTO_DIR}/type.proto + ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto +) + +#create directory first +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie) + +# proto gen for gie proto +protobuf_generate(APPEND_PATH + TARGET ${LOCAL_EXE_NAME} + LANGUAGE cpp + OUT_VAR PROTO_SRCS_GIE + PROTOS ${CODE_GEN_PROTOBUF_FILES} + IMPORT_DIRS ${GIE_COMPILER_PROTO_DIR} + PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/flex/proto_generated_gie +) + +add_library(ir_proto SHARED ${PROTO_SRCS_GIE}) + +target_link_libraries(ir_proto ${Protobuf_LIBRARIES}) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +file(GLOB_RECURSE PLANNER_SRC_FILES "graph_planner.cc") +add_library(graph_planner SHARED ${PLANNER_SRC_FILES}) +target_link_libraries(graph_planner PUBLIC ir_proto ${CMAKE_JNI_LINKER_FLAGS} ${JAVA_JVM_LIBRARY} ${JNI_LIBRARIES} ${Protobuf_LIBRARIES}) +target_include_directories(graph_planner PUBLIC $) + +if (BUILD_TEST) + add_executable(test_graph_planner test/test.cc) + target_link_libraries(test_graph_planner PRIVATE graph_planner) +endif() diff --git a/interactive_engine/compiler/src/main/native/README.md b/interactive_engine/compiler/src/main/native/README.md new file mode 100644 index 000000000000..f278ffe9433b --- /dev/null +++ b/interactive_engine/compiler/src/main/native/README.md @@ -0,0 +1,221 @@ +# Using the Graph Planner JNI Interface for C++ Invocation + +Follow the steps below to get started with the Graph Planner JNI interface for c++ invocation. + +## Getting Started +### Step 1: Build the Project + +Navigate to the project directory and build the package using Maven: +```bash +cd interactive_engine +mvn clean package -DskipTests -Pgraph-planner-jni +``` + +### Step 2: Locate and Extract the Package + +After the build completes, a tarball named `graph-planner-jni.tar.gz` will be available in the `assembly/target` directory. Extract the contents of the tarball: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +### Step 3: Run the Example Binary + +To demonstrate the usage of the JNI interface, an example binary `test_graph_planner` is provided. Use the following command to execute it: + +```bash +# bin/test_graph_planner +bin/test_graph_planner libs native ./conf/graph.yaml ./conf/modern_statistics.json "MATCH (n) RETURN n, COUNT(n);" ./conf/interactive_config_test.yaml +``` + +The output includes the physical plan and result schema in YAML format. Below is an example of a result schema: + +```yaml +schema: + name: default + description: default desc + mode: READ + extension: .so + library: libdefault.so + params: [] +returns: + - name: n + type: {primitive_type: DT_UNKNOWN} + - name: $f1 + type: {primitive_type: DT_SIGNED_INT64} +type: UNKNOWN +query: MATCH (n) RETURN n, COUNT(n); +``` + +The `returns` field defines the result schema. Each entry’s name specifies the column name, and the order of each entry determines the column IDs. + +## Explanation + +Below is a brief explanation of the interface provided by the example: + +### Constructor + +```cpp +/** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) + */ +GraphPlannerWrapper(const std::string &java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml = "", + const std::string &graph_statistic_json = ""); +``` + +## Method + +```cpp +/** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format + * @return The physical plan in bytes and result schema in yaml. + */ +Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) +``` + +Here is a refined version of your documentation with improvements for clarity, consistency, and readability: + +## Restful API + +We provide an alternative method to expose the interface as a RESTful API. Follow the steps below to access the interface via REST. + +### Step 1: Build the Project + +To build the project, run the following command: +```bash +cd interactive_engine +# Use '-Dskip.native=true' to skip compiling C++ native code +mvn clean package -DskipTests -Pgraph-planner-jni -Dskip.native=true +``` + +### Step 2: Locate and Extract the Package + +Once the build completes, a tarball named graph-planner-jni.tar.gz will be available in the assembly/target directory. Extract the contents as follows: + +```bash +cd assembly/target +tar xvzf graph-planner-jni.tar.gz +cd graph-planner-jni +``` + +### Step 3: Start the Graph Planner RESTful Service + +To start the service, run the following command: + +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.restful.GraphPlannerService --spring.config.location=./conf/application.yaml +``` + +### Step 4: Access the RESTful API + +To send a request to the RESTful API, use the following `curl` command: + +```bash +curl -X POST http://localhost:8080/api/compilePlan \ + -H "Content-Type: application/json" \ + -d "{ + \"configPath\": \"$configPath\", + \"query\": \"$query\", + \"schemaYaml\": \"$schemaYaml\", + \"statsJson\": \"$statsJson\" + }" +``` + +Replace `$configPath`, `$query`, `$schemaYaml`, and `$statsJson` with the appropriate values. + +The response will be in JSON format, similar to: + +```json +{ + "graphPlan": { + "physicalBytes": "", + "resultSchemaYaml": "" + } +} +``` + +The response contains two fields: +1. physicalBytes: A Base64-encoded string representing the physical plan bytes. +2. resultSchemaYaml: A string representing the YAML schema. + +You can decode these values into the required structures. + +Alternatively, you can run the following `Java` command to execute the same request: + +```bash +java -cp ".:./libs/*" com.alibaba.graphscope.sdk.examples.TestGraphPlanner ./conf/interactive_config_test.yaml "Match (n) Return n;" ./conf/graph.yaml ./conf/modern_statistics.json +``` + +Here’s an example of how to access the RESTful API and decode the response in Java code: +``` +public static void main(String[] args) throws Exception { + if (args.length < 4) { + System.out.println("Usage: "); + System.exit(1); + } + // set request body in json format + String jsonPayLoad = createParameters(args[0], args[1], args[2], args[3]).toString(); + HttpClient client = HttpClient.newBuilder().build(); + // create http request, set header and body content + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + // send request and get response + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + // parse response body as json + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + // print result + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); +} + +private static JsonNode createParameters( + String configPath, String query, String schemaPath, String statsPath) throws Exception { + Map params = + ImmutableMap.of( + "configPath", + configPath, + "query", + query, + "schemaYaml", + FileUtils.readFileToString(new File(schemaPath), StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString(new File(statsPath), StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); +} + +// get base64 string from json, convert it to physical bytes , then parse it to PhysicalPlan +private static GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) + throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); +} + +// get result schema yaml from json +private static String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); +} +``` + \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/native/graph_planner.cc b/interactive_engine/compiler/src/main/native/graph_planner.cc new file mode 100644 index 000000000000..d5dbbba57973 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.cc @@ -0,0 +1,464 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include +#include +#include +#include +#include +#include + +#include "graph_planner.h" +#include + +namespace gs +{ +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + + static JavaVM *_jvm = NULL; + + JavaVM *CreateJavaVM(const std::string &jvm_options) + { + const char *p, *q; + const char *jvm_opts; + if (jvm_options.empty()) + { + jvm_opts = getenv("FLEX_JVM_OPTS"); + } + else + { + jvm_opts = jvm_options.c_str(); + } + if (jvm_opts == NULL) + { + std::cerr << "Expect FLEX_JVM_OPTS set before initiate jvm" << std::endl; + return NULL; + } + std::cout << "Jvm opts str: " << jvm_opts << std::endl; + + if (*jvm_opts == '\0') + return NULL; + + int num_of_opts = 1; + for (const char *p = jvm_opts; *p; p++) + { + if (*p == ' ') + num_of_opts++; + } + + if (num_of_opts == 0) + return NULL; + + JavaVM *jvm = NULL; + JNIEnv *env = NULL; + int i = 0; + int status = 1; + JavaVMInitArgs vm_args; + + JavaVMOption *options = new JavaVMOption[num_of_opts]; + memset(options, 0, sizeof(JavaVMOption) * num_of_opts); + + for (p = q = jvm_opts;; p++) + { + if (*p == ' ' || *p == '\0') + { + if (q >= p) + { + goto ret; + } + char *opt = new char[p - q + 1]; + memcpy(opt, q, p - q); + opt[p - q] = '\0'; + options[i++].optionString = opt; + q = p + 1; // assume opts are separated by single space + if (*p == '\0') + break; + } + } + + memset(&vm_args, 0, sizeof(vm_args)); + vm_args.version = JNI_VERSION_1_8; + vm_args.nOptions = num_of_opts; + vm_args.options = options; + + status = JNI_CreateJavaVM(&jvm, reinterpret_cast(&env), &vm_args); + if (status == JNI_OK) + { + std::cout << "Create java virtual machine successfully." << std::endl; + } + else if (status == JNI_EEXIST) + { + std::cout << "JNI evn already exists." << std::endl; + } + else + { + std::cerr << "Error, create java virtual machine failed. return JNI_CODE (" + << status << ")" << std::endl; + } + + ret: + for (int i = 0; i < num_of_opts; i++) + { + delete[] options[i].optionString; + } + delete[] options; + return jvm; + } + + // One process can only create jvm for once. + JavaVM *GetJavaVM(const std::string jvm_options = "") + { + if (_jvm == NULL) + { + // Try to find whether there exists one javaVM + jsize nVMs; + JNI_GetCreatedJavaVMs(NULL, 0, + &nVMs); // 1. just get the required array length + std::cout << "Found " << nVMs << " VMs existing in this process." + << std::endl; + JavaVM **buffer = new JavaVM *[nVMs]; + JNI_GetCreatedJavaVMs(buffer, nVMs, &nVMs); // 2. get the data + for (auto i = 0; i < nVMs; ++i) + { + if (buffer[i] != NULL) + { + _jvm = buffer[i]; + std::cout << "Found index " << i << " VM non null " + << reinterpret_cast(_jvm) << std::endl; + return _jvm; + } + } + _jvm = CreateJavaVM(jvm_options); + std::cout << "Created JVM " << reinterpret_cast(_jvm) << std::endl; + } + return _jvm; + } + + JNIEnvMark::JNIEnvMark() : JNIEnvMark::JNIEnvMark("") {} + + JNIEnvMark::JNIEnvMark(const std::string &jvm_options) : _env(NULL) + { + if (!GetJavaVM(jvm_options)) + { + return; + } + int status = + GetJavaVM(jvm_options) + ->AttachCurrentThread(reinterpret_cast(&_env), nullptr); + if (status != JNI_OK) + { + std::cerr << "Error attach current thread: " << status << std::endl; + } + } + + JNIEnvMark::~JNIEnvMark() + { + if (_env) + { + GetJavaVM()->DetachCurrentThread(); + } + } + + JNIEnv *JNIEnvMark::env() { return _env; } + + } // namespace jni + +#endif + + std::vector list_files(const std::string &path) + { + // list all files in the directory + std::vector files; + for (const auto &entry : std::filesystem::directory_iterator(path)) + { + files.push_back(entry.path().string()); + } + return files; + } + + void iterate_over_director(const std::string &dir_or_path, std::vector &output_paths) + { + if (dir_or_path.empty()) + { + return; + } + if (std::filesystem::is_directory(dir_or_path)) + { + auto files = list_files(dir_or_path); + output_paths.insert(output_paths.end(), files.begin(), files.end()); + } + else + { + output_paths.push_back(dir_or_path); + } + } + + std::string GraphPlannerWrapper::expand_directory(const std::string &path) + { + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + iterate_over_director(sub_path, paths); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + iterate_over_director(sub_path, paths); + std::stringstream ss; + for (const auto &p : paths) + { + ss << p << ":"; + } + return ss.str(); + } + +#if (GRAPH_PLANNER_JNI_INVOKER) + + std::string GraphPlannerWrapper::generate_jvm_options( + const std::string java_path, const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { + auto expanded_java_path = expand_directory(java_path); + std::cout << "Expanded java path: " << expanded_java_path << std::endl; + std::string jvm_options = "-Djava.class.path=" + expanded_java_path; + jvm_options += " -Djna.library.path=" + jna_path; + // jvm_options += " -Dgraph.schema=" + graph_schema_yaml; + // if (!graph_statistic_json.empty()) + // { + // jvm_options += " -Dgraph.statistics=" + graph_statistic_json; + // } + return jvm_options; + } + + Plan compilePlanJNI(jclass graph_planner_clz_, + jmethodID graph_planner_method_id_, JNIEnv *env, + const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { + jni::GetJavaVM()->AttachCurrentThread(reinterpret_cast(&env), + nullptr); + Plan plan; + if (graph_planner_clz_ == NULL || graph_planner_method_id_ == NULL) + { + std::cerr << "Invalid GraphPlannerWrapper." << std::endl; + return plan; + } + jstring param1 = env->NewStringUTF(compiler_config_path.c_str()); + jstring param2 = env->NewStringUTF(cypher_query_string.c_str()); + jstring param3 = env->NewStringUTF(graph_schema_yaml.c_str()); + jstring param4 = env->NewStringUTF(graph_statistic_json.c_str()); + + // invoke jvm static function to get results as Object[] + jobject jni_plan = (jobject)env->CallStaticObjectMethod( + graph_planner_clz_, graph_planner_method_id_, param1, param2, param3, param4); + + if (env->ExceptionCheck()) + { + env->ExceptionDescribe(); + env->ExceptionClear(); + std::cerr << "Error in calling GraphPlanner." << std::endl; + return plan; + } + + jmethodID method1 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getPhysicalBytes", "()[B"); + jmethodID method2 = env->GetMethodID( + env->GetObjectClass(jni_plan), "getResultSchemaYaml", "()Ljava/lang/String;"); + + // 0-th object is the physical plan in byte array + jbyteArray res1 = (jbyteArray)env->CallObjectMethod(jni_plan, method1); + // 1-th object is the result schema in yaml format + jstring res2 = (jstring)env->CallObjectMethod(jni_plan, method2); + + if (res1 == NULL || res2 == NULL) + { + std::cerr << "Fail to generate plan." << std::endl; + return plan; + } + jbyte *str = env->GetByteArrayElements(res1, NULL); + jsize len = env->GetArrayLength(res1); + std::cout << "Physical plan size: " << len; + + plan.physical_plan.ParseFromArray(str, len); + plan.result_schema = env->GetStringUTFChars(res2, NULL); + + env->ReleaseByteArrayElements(res1, str, 0); + env->DeleteLocalRef(param1); + env->DeleteLocalRef(param2); + env->DeleteLocalRef(res1); + // remove new added jni objects + env->DeleteLocalRef(res2); + env->DeleteLocalRef(jni_plan); + + return plan; + } +#endif + +#if (!GRAPH_PLANNER_JNI_INVOKER) + + void write_query_to_pipe(const std::string &path, + const std::string &query_str) + { + std::cout << "write_query_to_pipe: " << path << std::endl; + + // mkfifo(path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + int fd_to_java = open(path.c_str(), O_WRONLY); + if (fd_to_java < 0) + { + std::cerr << "Fail to open pipe: " << path << std::endl; + return; + } + std::cout << "open pipe done" << std::endl; + auto len = write(fd_to_java, query_str.c_str(), query_str.size()); + if (len != (int)query_str.size()) + { + std::cerr << "Fail to write query to pipe:" << len << std::endl; + return; + } + std::cout << "write_query_to_pipe done: " << len << std::endl; + close(fd_to_java); + } + + void write_query_to_file(const std::string &path, + const std::string &query_str) + { + std::ofstream query_file(path); + query_file << query_str; + query_file.close(); + } + + physical::PhysicalPlan readPhysicalPlan(const std::string &plan_str) + { + std::cout << "plan str size: " << plan_str.size() << std::endl; + physical::PhysicalPlan plan; + if (!plan.ParseFromString(plan_str)) + { + std::cerr << "Fail to parse physical plan." << std::endl; + return physical::PhysicalPlan(); + } + return plan; + } + + physical::PhysicalPlan + compilePlanSubprocess(const std::string &class_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json, + const std::string &compiler_config_path, + const std::string &cypher_query_string) + { + physical::PhysicalPlan physical_plan; + auto random_prefix = std::to_string( + std::chrono::system_clock::now().time_since_epoch().count()); + std::string dst_query_path = "/tmp/temp_query_" + random_prefix + ".cypher"; + std::string dst_output_file = "/tmp/temp_output_" + random_prefix + ".pb"; + std::cout << "dst_query_path: " << dst_query_path + << " dst_output_file: " << dst_output_file << std::endl; + mkfifo(dst_query_path.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + mkfifo(dst_output_file.c_str(), S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH); + + pid_t pid = fork(); + + if (pid == 0) + { + const char *const command_string_array[] = {"java", + "-cp", + class_path.c_str(), + jna_path.c_str(), + graph_schema_yaml.c_str(), + graph_statistic_json.c_str(), + GRAPH_PLANNER_FULL_NAME, + compiler_config_path.c_str(), + dst_query_path.c_str(), + dst_output_file.c_str(), + "/tmp/temp.cypher.yaml", + NULL}; + execvp(command_string_array[0], + const_cast(command_string_array)); + } + else if (pid < 0) + { + std::cerr << "Error in fork." << std::endl; + } + else + { + write_query_to_pipe(dst_query_path, cypher_query_string); + + int fd_from_java = open(dst_output_file.c_str(), O_RDONLY); + if (fd_from_java < 0) + { + std::cerr << "Fail to open pipe: " << dst_output_file << std::endl; + return physical_plan; + } + std::vector stored_buffer; + char buffer[128]; + while (true) + { + ssize_t bytesRead = read(fd_from_java, buffer, sizeof(buffer) - 1); + if (bytesRead <= 0) + { + break; + } + stored_buffer.insert(stored_buffer.end(), buffer, buffer + bytesRead); + } + physical_plan = readPhysicalPlan( + std::string(stored_buffer.begin(), stored_buffer.end())); + close(fd_from_java); + + int status; + waitpid(pid, &status, 0); + if (status != 0) + { + std::cerr << "Error in running command." << std::endl; + } + } + unlink(dst_query_path.c_str()); + unlink(dst_output_file.c_str()); + return physical_plan; + } +#endif + + /** + * @brief Compile a cypher query to a physical plan by JNI invocation. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @param graph_schema_yaml Content of the graph schema in YAML format + * @param graph_statistic_json Content of the graph statistics in JSON format + * @return The physical plan in bytes and result schema in yaml. + */ + Plan GraphPlannerWrapper::CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json) + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return compilePlanJNI(graph_planner_clz_, graph_planner_method_id_, + jni_wrapper_.env(), compiler_config_path, + cypher_query_string, graph_schema_yaml, graph_statistic_json); +#else + return compilePlanSubprocess(class_path_, jna_path_, graph_schema_yaml_, + graph_statistic_json_, compiler_config_path, + cypher_query_string); +#endif + } + +} // namespace gs diff --git a/interactive_engine/compiler/src/main/native/graph_planner.h b/interactive_engine/compiler/src/main/native/graph_planner.h new file mode 100644 index 000000000000..40de6b3d0161 --- /dev/null +++ b/interactive_engine/compiler/src/main/native/graph_planner.h @@ -0,0 +1,169 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#ifndef PLANNER_GRAPH_PLANNER_H_ +#define PLANNER_GRAPH_PLANNER_H_ + +#include +#include +#include +#include +#include +#include + +#include "flex/proto_generated_gie/physical.pb.h" + +#ifndef GRAPH_PLANNER_JNI_INVOKER +#define GRAPH_PLANNER_JNI_INVOKER 1 // 1: JNI, 0: subprocess +#endif + +namespace gs +{ + + struct Plan + { + physical::PhysicalPlan physical_plan; + std::string result_schema; + }; + +#if (GRAPH_PLANNER_JNI_INVOKER) + namespace jni + { + struct JNIEnvMark + { + JNIEnv *_env; + + JNIEnvMark(); + JNIEnvMark(const std::string &jvm_options); + ~JNIEnvMark(); + JNIEnv *env(); + }; + + } // namespace jni +#endif + + class GraphPlannerWrapper + { + public: + static constexpr const char *kGraphPlannerClass = + "com/alibaba/graphscope/sdk/PlanUtils"; + static constexpr const char *GRAPH_PLANNER_FULL_NAME = + "com.alibaba.graphscope.sdk.PlanUtils"; + static constexpr const char *kGraphPlannerMethod = "compilePlan"; + static constexpr const char *kGraphPlannerMethodSignature = + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Lcom/alibaba/graphscope/sdk/GraphPlan;"; + + /** + * @brief Constructs a new GraphPlannerWrapper object + * @param java_path Java class path + * @param jna_path JNA library path + * @param graph_schema_yaml Path to the graph schema file in YAML format (optional) + * @param graph_statistic_json Path to the graph statistics file in JSON format (optional) + */ + GraphPlannerWrapper(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml = "", + const std::string &graph_statistic_json = "") +#if (GRAPH_PLANNER_JNI_INVOKER) + : jni_wrapper_(generate_jvm_options( + java_path, jna_path, graph_schema_yaml, graph_statistic_json)) + { + jclass clz = jni_wrapper_.env()->FindClass(kGraphPlannerClass); + if (clz == NULL) + { + std::cerr << "Fail to find class: " << kGraphPlannerClass << std::endl; + return; + } + graph_planner_clz_ = (jclass)jni_wrapper_.env()->NewGlobalRef(clz); + jmethodID j_method_id = jni_wrapper_.env()->GetStaticMethodID( + graph_planner_clz_, kGraphPlannerMethod, kGraphPlannerMethodSignature); + if (j_method_id == NULL) + { + std::cerr << "Fail to find method: " << kGraphPlannerMethod << std::endl; + return; + } + graph_planner_method_id_ = j_method_id; + } +#else + : jna_path_("-Djna.library.path=" + jna_path), + graph_schema_yaml_("-Dgraph.schema=" + graph_schema_yaml), + graph_statistic_json_("-Dgraph.statistics=" + graph_statistic_json) + { + class_path_ = expand_directory(java_path); + } +#endif + + ~GraphPlannerWrapper() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + if (graph_planner_clz_ != NULL) + { + jni_wrapper_.env()->DeleteGlobalRef(graph_planner_clz_); + } +#endif + } + + inline bool is_valid() + { +#if (GRAPH_PLANNER_JNI_INVOKER) + return graph_planner_clz_ != NULL && graph_planner_method_id_ != NULL; +#else + return true; // just return true, since we don't have a way to check the + // validity when calling via subprocess. +#endif + } + + /** + * @brief Invoker GraphPlanner to generate a physical plan from a cypher + * query. + * @param compiler_config_path The path of compiler config file. + * @param cypher_query_string The cypher query string. + * @return physical plan in string. + */ + Plan CompilePlan(const std::string &compiler_config_path, + const std::string &cypher_query_string, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); + + private: + std::string generate_jvm_options(const std::string java_path, + const std::string &jna_path, + const std::string &graph_schema_yaml, + const std::string &graph_statistic_json); + // physical::PhysicalPlan compilePlanJNI(const std::string& + // compiler_config_path, + // const std::string& + // cypher_query_string); + std::string expand_directory(const std::string &path); +#if (GRAPH_PLANNER_JNI_INVOKER) + // We need to list all files in the directory, if exists. + // The reason why we need to list all files in the directory is that + // java -Djava.class.path=dir/* (in jni, which we are using)will not load all + // jar files in the directory, While java -cp dir/* will load all jar files in + // the directory. + + gs::jni::JNIEnvMark jni_wrapper_; + jclass graph_planner_clz_; + jmethodID graph_planner_method_id_; +#else + std::string class_path_; + std::string jna_path_; + std::string graph_schema_yaml_; + std::string graph_statistic_json_; +#endif + }; +} // namespace gs + +#endif // PLANNER_GRAPH_PLANNER_H_ diff --git a/interactive_engine/compiler/src/main/native/test/test.cc b/interactive_engine/compiler/src/main/native/test/test.cc new file mode 100644 index 000000000000..1c58f11af6eb --- /dev/null +++ b/interactive_engine/compiler/src/main/native/test/test.cc @@ -0,0 +1,121 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#include "graph_planner.h" + +#include +#include +#include + +#include + +std::string get_dir_name() +{ + // Get the directory of this source file + std::string current_dir = __FILE__; + size_t pos = current_dir.find_last_of("/"); + current_dir = current_dir.substr(0, pos); + return current_dir; +} + +void check_path_exits(const std::string &path) +{ + // split path by ':' + std::vector paths; + std::string::size_type start = 0; + std::string::size_type end = path.find(':'); + while (end != std::string::npos) + { + auto sub_path = path.substr(start, end - start); + paths.push_back(sub_path); + start = end + 1; + end = path.find(':', start); + } + auto sub_path = path.substr(start); + paths.push_back(sub_path); + + for (const auto &p : paths) + { + struct stat buffer; + if (stat(p.c_str(), &buffer) != 0) + { + std::cerr << "Path not exists: " << p << std::endl; + exit(1); + } + } + std::cout << "Path exists: " << path << std::endl; +} + +std::string read_string_from_file(const std::string &file_path) +{ + std::ifstream inputFile(file_path); // Open the file for reading + + if (!inputFile.is_open()) + { + std::cerr << "Error: Could not open the file " << file_path << std::endl; + exit(1); + } + // Use a stringstream to read the entire content of the file + std::ostringstream buffer; + buffer << inputFile.rdbuf(); // Read the file stream into the stringstream + + std::string fileContent = buffer.str(); // Get the string from the stringstream + inputFile.close(); // Close the file + + return fileContent; +} + +int main(int argc, char **argv) +{ + // Check if the correct number of arguments is provided + if (argc != 7) + { + std::cerr << "Usage: " << argv[0] + << " " << std::endl; + return 1; + } + + // auto start = std::chrono::high_resolution_clock::now(); + + std::string java_class_path = argv[1]; + std::string jna_class_path = argv[2]; + std::string graph_schema_path = argv[3]; + std::string graph_statistic_path = argv[4]; + + // check director or file exists + check_path_exits(java_class_path); + check_path_exits(jna_class_path); + check_path_exits(graph_schema_path); + check_path_exits(graph_statistic_path); + + gs::GraphPlannerWrapper graph_planner_wrapper( + java_class_path, jna_class_path); + + std::string schema_content = read_string_from_file(graph_schema_path); + std::string statistic_content = read_string_from_file(graph_statistic_path); + + std::string cypher_query_string = argv[5]; + std::string config_path = argv[6]; + auto plan = + graph_planner_wrapper.CompilePlan(config_path, cypher_query_string, schema_content, statistic_content); + + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start).count(); + + // std::cout << "total execution time: " << duration << " ms" << std::endl; + + std::cout << "Plan: " << plan.physical_plan.DebugString() << std::endl; + std::cout << "schema: " << plan.result_schema << std::endl; + return 0; +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java new file mode 100644 index 000000000000..cadc5dbc5649 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/JNICompilePlanTest.java @@ -0,0 +1,49 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; + +public class JNICompilePlanTest { + @Test + public void planQuery() throws Exception { + String configPath = "src/test/resources/config/interactive_config_test.yaml"; + String query = + "MATCH (src)-[e:test6*4..5]->(dest) WHERE src.__domain__ = 'xzz' RETURN" + + " src.__entity_id__ AS sId, dest.__entity_id__ AS dId;"; + String schemaYaml = + FileUtils.readFileToString( + new File("/Users/zhouxiaoli/Downloads/graph_schema.yaml"), + StandardCharsets.UTF_8); + String statsJson = + FileUtils.readFileToString( + new File("/Users/zhouxiaoli/Downloads/statistics.json"), + StandardCharsets.UTF_8); + for (int i = 0; i < 100; ++i) { + long startTime = System.currentTimeMillis(); + PlanUtils.compilePlan(configPath, query, schemaYaml, statsJson); + long endTime = System.currentTimeMillis() - startTime; + System.out.println("Time cost: " + endTime); + } + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java new file mode 100644 index 000000000000..be5418d2141a --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/sdk/RestCompilePlanTest.java @@ -0,0 +1,86 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.sdk; + +import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class RestCompilePlanTest { + @Test + public void testCompilePlan() throws Exception { + String jsonPayLoad = createParameters().toString(); + HttpClient client = HttpClient.newBuilder().build(); + System.out.println(jsonPayLoad); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8080/api/compilePlan")) + .setHeader("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + jsonPayLoad, StandardCharsets.UTF_8)) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + System.out.println(body); + JsonNode planNode = (new ObjectMapper()).readTree(body).get("graphPlan"); + System.out.println(getPhysicalPlan(planNode)); + System.out.println(getResultSchemaYaml(planNode)); + } + + private JsonNode createParameters() throws Exception { + Map params = + ImmutableMap.of( + "configPath", + "conf/ir.compiler.properties", + "query", + "Match (n) Return n;", + "schemaYaml", + FileUtils.readFileToString( + new File("src/test/resources/config/modern/graph.yaml"), + StandardCharsets.UTF_8), + "statsJson", + FileUtils.readFileToString( + new File("src/test/resources/statistics/modern_statistics.json"), + StandardCharsets.UTF_8)); + return (new ObjectMapper()).valueToTree(params); + } + + private GraphAlgebraPhysical.PhysicalPlan getPhysicalPlan(JsonNode planNode) throws Exception { + String base64Str = planNode.get("physicalBytes").asText(); + byte[] bytes = java.util.Base64.getDecoder().decode(base64Str); + return GraphAlgebraPhysical.PhysicalPlan.parseFrom(bytes); + } + + private String getResultSchemaYaml(JsonNode planNode) { + return planNode.get("resultSchemaYaml").asText(); + } +} diff --git a/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml b/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml new file mode 100644 index 000000000000..84ac110dffb3 --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/config/interactive_config_test.yaml @@ -0,0 +1,46 @@ +log_level: INFO +verbose_level: 10 +default_graph: modern_graph +compute_engine: + type: hiactor + workers: + - localhost:10000 + thread_num_per_worker: 1 + store: + type: cpp-mcsr + metadata_store: + type: file # file/sqlite/etcd +compiler: + physical: + opt: + config: proto + planner: + is_on: true + opt: CBO + rules: + - FilterIntoJoinRule + - FilterMatchRule + - ExtendIntersectRule + - ExpandGetVFusionRule + meta: + reader: + schema: + uri: http://localhost:7777/v1/service/status + interval: 1000 # ms + statistics: + uri: http://localhost:7777/v1/graph/%s/statistics + interval: 86400000 # ms + endpoint: + default_listen_address: localhost + bolt_connector: + disabled: false + port: 7687 + gremlin_connector: + disabled: false + port: 8182 + query_timeout: 40000 + gremlin_script_language_name: antlr_gremlin_calcite +http_service: + default_listen_address: localhost + admin_port: 7777 + query_port: 10000 diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index f69d68faca14..f89dd1754e40 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -50,21 +50,45 @@ - graphscope - - true - + graph-planner-jni - v6d + false + + false + - assembly common - executor - frontend executor/engine/pegasus/clients/java/client compiler + assembly + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + + run + + compile + + + + + + + ${skip.native} + + + + false + + + groot @@ -698,6 +722,11 @@ ${interactive.sdk.version} ${interactive.sdk.classifier} + + org.springframework.boot + spring-boot-starter-web + 2.2.4.RELEASE + @@ -885,6 +914,11 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 3.0.0 +