From 790cb1218f0b16e733727d4ff9d17cb0d7dc243a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 14 Dec 2023 19:47:39 +0800 Subject: [PATCH 1/2] [GIE Compiler] fix bugs of array types as stored procedure arguments --- .../ir/runtime/ProcedurePhysicalBuilder.java | 21 +++-- .../ir/runtime/proto/RexToProtoConverter.java | 15 ++++ .../common/ir/runtime/proto/Utils.java | 80 +++++++++++++++++++ .../common/ir/tools/GraphPlanner.java | 2 +- .../cypher/result/CypherRecordParser.java | 19 +++-- .../common/ir/runtime/FfiLogicalPlanTest.java | 3 +- 6 files changed, 127 insertions(+), 13 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ProcedurePhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ProcedurePhysicalBuilder.java index a7b2b3dda339..b1277f1cf1da 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ProcedurePhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ProcedurePhysicalBuilder.java @@ -16,8 +16,11 @@ package com.alibaba.graphscope.common.ir.runtime; -import com.alibaba.graphscope.common.ir.runtime.proto.Utils; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.Common; import com.alibaba.graphscope.gaia.proto.StoredProcedure; import com.google.protobuf.InvalidProtocolBufferException; @@ -31,12 +34,18 @@ public class ProcedurePhysicalBuilder extends PhysicalBuilder { private final StoredProcedure.Query.Builder builder; - public ProcedurePhysicalBuilder(LogicalPlan logicalPlan) { + public ProcedurePhysicalBuilder(Configs configs, IrMeta irMeta, LogicalPlan logicalPlan) { super(logicalPlan); this.builder = StoredProcedure.Query.newBuilder(); RexCall procedureCall = (RexCall) logicalPlan.getProcedureCall(); setStoredProcedureName(procedureCall, builder); - setStoredProcedureArgs(procedureCall, builder); + setStoredProcedureArgs( + procedureCall, + builder, + new RexToProtoConverter( + true, + irMeta.getSchema().isColumnId(), + GraphPlanner.rexBuilderFactory.apply(configs))); } private void setStoredProcedureName( @@ -46,14 +55,16 @@ private void setStoredProcedureName( } private void setStoredProcedureArgs( - RexCall procedureCall, StoredProcedure.Query.Builder builder) { + RexCall procedureCall, + StoredProcedure.Query.Builder builder, + RexToProtoConverter converter) { List operands = procedureCall.getOperands(); for (int i = 0; i < operands.size(); ++i) { builder.addArguments( StoredProcedure.Argument.newBuilder() // param name is omitted .setParamInd(i) - .setValue(Utils.protoValue((RexLiteral) operands.get(i))) + .setValue(operands.get(i).accept(converter).getOperators(0).getConst()) .build()); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java index b282b265f573..da11877926a9 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java @@ -28,6 +28,7 @@ import org.apache.calcite.rex.*; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.ArraySqlType; import org.apache.calcite.util.Sarg; import java.util.List; @@ -88,6 +89,20 @@ private OuterExpression.Expression visitCase(RexCall call) { } private OuterExpression.Expression visitArrayValueConstructor(RexCall call) { + // convert to literal if operands have the same primitive type + if (call.getOperands().stream().allMatch(k -> k instanceof RexLiteral) + && call.getType() instanceof ArraySqlType) { + return OuterExpression.Expression.newBuilder() + .addOperators( + OuterExpression.ExprOpr.newBuilder() + .setConst( + Utils.protoValue( + call.getOperands(), + ((ArraySqlType) call.getType()) + .getComponentType())) + .build()) + .build(); + } OuterExpression.VariableKeys.Builder varsBuilder = OuterExpression.VariableKeys.newBuilder(); call.getOperands() diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index c816b461302b..ce713dfe83df 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -31,6 +31,7 @@ import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.NlsString; @@ -116,6 +117,85 @@ public static final Common.Value protoValue(RexLiteral literal) { } } + public static final Common.Value protoValue(List literals, RelDataType literalType) { + switch (literalType.getSqlTypeName()) { + case INTEGER: + return Common.Value.newBuilder() + .setI32Array( + Common.I32Array.newBuilder() + .addAllItem( + literals.stream() + .map( + k -> + ((Number) + ((RexLiteral) + k) + .getValue()) + .intValue()) + .collect(Collectors.toList()))) + .build(); + case BIGINT: + return Common.Value.newBuilder() + .setI64Array( + Common.I64Array.newBuilder() + .addAllItem( + literals.stream() + .map( + k -> + ((Number) + ((RexLiteral) + k) + .getValue()) + .longValue()) + .collect(Collectors.toList()))) + .build(); + case FLOAT: + case DOUBLE: + return Common.Value.newBuilder() + .setF64Array( + Common.DoubleArray.newBuilder() + .addAllItem( + literals.stream() + .map( + k -> + ((Number) + ((RexLiteral) + k) + .getValue()) + .doubleValue()) + .collect(Collectors.toList()))) + .build(); + case CHAR: + return Common.Value.newBuilder() + .setStrArray( + Common.StringArray.newBuilder() + .addAllItem( + literals.stream() + .map( + k -> { + RexLiteral literal = + (RexLiteral) k; + if (literal.getValue() + instanceof NlsString) { + return ((NlsString) + literal + .getValue()) + .getValue(); + } else { + return (String) + literal.getValue(); + } + }) + .collect(Collectors.toList()))) + .build(); + default: + throw new UnsupportedOperationException( + "list of literal type " + + literalType.getSqlTypeName() + + " can not be converted to ir core array"); + } + } + public static final OuterExpression.Property protoProperty(GraphProperty property) { switch (property.getOpt()) { case ID: diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 18fdbb9a9b05..226eb9d5a8bf 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -161,7 +161,7 @@ public PhysicalPlan planPhysical(LogicalPlan logicalPlan) { throw new RuntimeException(e); } } else { - return new ProcedurePhysicalBuilder(logicalPlan).build(); + return new ProcedurePhysicalBuilder(graphConfig, irMeta, logicalPlan).build(); } } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java index c8c3f65ef09a..b60338e371ab 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java @@ -84,12 +84,19 @@ protected AnyValue parseEntry(IrResult.Entry entry, @Nullable RelDataType dataTy switch (dataType.getSqlTypeName()) { case MULTISET: case ARRAY: - if (dataType instanceof ArbitraryArrayType) { - return parseCollection( - entry.getCollection(), - ((ArbitraryArrayType) dataType).getComponentTypes()); - } else { - return parseCollection(entry.getCollection(), dataType.getComponentType()); + switch (entry.getInnerCase()) { + case COLLECTION: + if (dataType instanceof ArbitraryArrayType) { + return parseCollection( + entry.getCollection(), + ((ArbitraryArrayType) dataType).getComponentTypes()); + } else { + return parseCollection( + entry.getCollection(), dataType.getComponentType()); + } + case ELEMENT: + default: + return parseElement(entry.getElement(), dataType.getComponentType()); } case MAP: if (dataType instanceof ArbitraryMapType) { diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java index 9be7065deffa..4241ecbfab24 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java @@ -154,7 +154,8 @@ public void logical_plan_4_test() throws Exception { LogicalPlan logicalPlan = com.alibaba.graphscope.cypher.antlr4.Utils.evalLogicalPlan( "Call ldbc_ic2(10l, 20120112l)"); - try (PhysicalBuilder ffiBuilder = new ProcedurePhysicalBuilder(logicalPlan)) { + try (PhysicalBuilder ffiBuilder = + new ProcedurePhysicalBuilder(getMockGraphConfig(), Utils.schemaMeta, logicalPlan)) { PhysicalPlan plan = ffiBuilder.build(); Assert.assertEquals( FileUtils.readJsonFromResource("call_procedure.json"), plan.explain()); From 0f36609639003eaad989db655f5c022f6cc89d48 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 15 Dec 2023 17:08:32 +0800 Subject: [PATCH 2/2] support accept string/int/long Array on hqps engine,currently only for c++ procedure --- .../graph_db/database/graph_db_session.cc | 24 +++++++++++++++++++ .../meta/procedure/StoredProcedureMeta.java | 9 +++++-- .../ir/meta/reader/LocalMetaDataReader.java | 4 ++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/flex/engines/graph_db/database/graph_db_session.cc b/flex/engines/graph_db/database/graph_db_session.cc index e90be0308188..47f527076ba3 100644 --- a/flex/engines/graph_db/database/graph_db_session.cc +++ b/flex/engines/graph_db/database/graph_db_session.cc @@ -36,6 +36,30 @@ void put_argment(Encoder& encoder, const query::Argument& argment) { case common::Value::kStr: encoder.put_string(value.str()); break; + case common::Value::kStrArray: + encoder.put_int(value.str_array().item_size()); + for (auto i = 0; i < value.str_array().item_size(); ++i) { + encoder.put_string(value.str_array().item(i)); + } + break; + case common::Value::kF64Array: + encoder.put_int(value.f64_array().item_size()); + for (auto i = 0; i < value.f64_array().item_size(); ++i) { + encoder.put_double(value.f64_array().item(i)); + } + break; + case common::Value::kI32Array: + encoder.put_int(value.i32_array().item_size()); + for (auto i = 0; i < value.i32_array().item_size(); ++i) { + encoder.put_int(value.i32_array().item(i)); + } + break; + case common::Value::kI64Array: + encoder.put_int(value.i64_array().item_size()); + for (auto i = 0; i < value.i64_array().item_size(); ++i) { + encoder.put_long(value.i64_array().item(i)); + } + break; default: LOG(ERROR) << "Not recognizable param type" << static_cast(item_case); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index 1e63748fb1a0..ac57e1e809e0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -17,11 +17,12 @@ package com.alibaba.graphscope.common.ir.meta.procedure; import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.ir.type.GraphTypeFactoryImpl; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.*; import org.yaml.snakeyaml.Yaml; @@ -33,7 +34,11 @@ import java.util.stream.Collectors; public class StoredProcedureMeta { - private static final RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + private static final RelDataTypeFactory typeFactory = + new GraphTypeFactoryImpl( + new Configs( + ImmutableMap.of( + FrontendConfig.CALCITE_DEFAULT_CHARSET.getKey(), "UTF-8"))); private final String name; private final RelDataType returnType; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java index b8ce6a9dd960..be057dc4a7f1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java @@ -76,6 +76,10 @@ public List getStoredProcedures() throws IOException { getProcedureNameWithInputStream(procedureDir); for (String enableProcedure : enableProcedureList) { InputStream enableInput = procedureInputMap.get(enableProcedure); + if (enableInput == null) { + logger.error("procedure not found {}", enableProcedure); + continue; + } Preconditions.checkArgument( enableInput != null, "can not find procedure with name=%s under directory=%s, candidates are %s",