fix(interactive): Implement HttpIrMetaReader to Get Meta Data From Remote Http Service

.github/workflows/interactive.yml

@@ -180,6 +180,20 @@ jobs:
         ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
         /tmp/csr-data-dir/
 
+    - name: Test get graph meta from admin service
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
+        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
+        sed -i 's/mode: local/mode: http/g' ./engine_config_test.yaml
+        pip3 install argparse
+        pip3 install neo4j >= 4.4.19
+        bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml
+        sed -i 's/mode: http/mode: local/g' ./engine_config_test.yaml
+        sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml
+
     - name: Run codegen test.
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest

flex/tests/hqps/engine_config_test.yaml

@@ -23,6 +23,13 @@ compiler:
       - FilterIntoJoinRule
       - FilterMatchRule
       - NotMatchToAntiJoinRule
+  meta:
+    reader:
+      mode: local
+      endpoint: localhost:7777 # only for http mode
+      interval:
+        graph_meta: 1000 # ms
+        statistics: 86400000 # ms
   endpoint:
     default_listen_address: localhost
     bolt_connector:

flex/tests/hqps/hqps_compiler_get_meta_test.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+# Copyright 2020 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+FLEX_HOME=${SCRIPT_DIR}/../../
+SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server
+GIE_HOME=${FLEX_HOME}/../interactive_engine/
+ADMIN_PORT=7777
+QUERY_PORT=10000
+
+# 
+if [ ! $# -eq 2 ]; then
+  echo "only receives: $# args, need 2"
+  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <ENGINE_CONFIG>"
+  exit 1
+fi
+
+INTERACTIVE_WORKSPACE=$1
+ENGINE_CONFIG_PATH=$2
+if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then
+  echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists"
+  exit 1
+fi
+if [ ! -f ${ENGINE_CONFIG_PATH} ]; then
+  echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists"
+  exit 1
+fi
+
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+err() {
+  echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2
+}
+
+info() {
+  echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}"
+}
+
+
+kill_service(){
+    info "Kill Service first"
+    ps -ef | grep "interactive_server" |  awk '{print $2}' | xargs kill -9  || true
+    ps -ef | grep "compiler" |  awk '{print $2}' | xargs kill -9  || true
+    sleep 3
+    # check if service is killed
+    info "Kill Service success"
+}
+
+# kill service when exit
+trap kill_service EXIT
+
+# start engine service and load ldbc graph
+start_engine_service(){
+    #check SERVER_BIN exists
+    if [ ! -f ${SERVER_BIN} ]; then
+        err "SERVER_BIN not found"
+        exit 1
+    fi
+
+    cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} --enable-admin-service true "
+    cmd="${cmd}  -w ${INTERACTIVE_WORKSPACE} --start-compiler true"
+
+    echo "Start engine service with command: ${cmd}"
+    eval ${cmd} 
+    sleep 10
+    #check interactive_server is running, if not, exit
+    ps -ef | grep "interactive_server" | grep -v grep
+
+    info "Start engine service success"
+}
+
+run_cypher_test() {
+  # run a simple cypher query: MATCH (n) RETURN count(n)
+  python3 ./test_count_vertices.py --endpoint localhost:7687
+}
+
+kill_service
+start_engine_service
+# comiper service will fail to start, if the graph meta can not be retrieved
+run_cypher_test
+kill_service
+
+
+
+

flex/tests/hqps/test_count_vertices.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from neo4j import GraphDatabase
+from neo4j import Session as Neo4jSession
+
+import argparse
+
+def count_vertices(sess: Neo4jSession):
+    query = "MATCH (n) RETURN COUNT(n);"
+    result = sess.run(query)
+    for record in result:
+        print(record[0])
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Count the number of vertices in a graph.")
+    parser.add_argument("--endpoint", type=str, required=True, help="The endpoint to connect.")
+    args = parser.parse_args()
+
+    driver = GraphDatabase.driver(args.endpoint, auth=None)
+    with driver.session() as session:
+        count_vertices(session)
+    driver.close()

interactive_engine/compiler/conf/ir.compiler.properties

@@ -61,3 +61,15 @@ calcite.default.charset: UTF-8
 
 # set the max capacity of the result streaming buffer for each query
 # per.query.stream.buffer.max.capacity: 256
+
+# set the mode to read ir meta, either from a 'local' file or via remote 'http'.
+# ir.meta.reader.mode: local
+
+# if ir.meta.reader.mode is 'http', set the remote ir meta service host
+# ir.meta.service.host: localhost:8080
+
+# if ir.meta.reader.mode is 'http', set the interval in milliseconds to fetch ir meta
+# ir.meta.fetch.interval.ms: 1000
+
+# if ir.meta.reader.mode is 'http', set the timeout in milliseconds to fetch ir graph statistics
+# ir.statistics.fetch.interval.ms: 86400000l

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java

@@ -20,13 +20,16 @@
 import com.alibaba.graphscope.common.client.channel.ChannelFetcher;
 import com.alibaba.graphscope.common.client.channel.HostURIChannelFetcher;
 import com.alibaba.graphscope.common.client.channel.HostsRpcChannelFetcher;
+import com.alibaba.graphscope.common.client.channel.MetaServiceChannelFetcher;
 import com.alibaba.graphscope.common.config.Configs;
 import com.alibaba.graphscope.common.config.FrontendConfig;
 import com.alibaba.graphscope.common.config.GraphConfig;
 import com.alibaba.graphscope.common.ir.meta.IrMeta;
 import com.alibaba.graphscope.common.ir.meta.IrMetaTracker;
+import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher;
 import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher;
 import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher;
+import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader;
 import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader;
 import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer;
 import com.alibaba.graphscope.common.ir.tools.*;
@@ -187,7 +190,18 @@ public static void main(String[] args) throws Exception {
 
     private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker)
             throws IOException {
-        return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker);
+        String readerMode = FrontendConfig.IR_META_READER_MODE.get(configs);
+        switch (readerMode) {
+            case "local":
+                return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker);
+            case "http":
+                return new DynamicIrMetaFetcher(
+                        configs,
+                        new HttpIrMetaReader(new MetaServiceChannelFetcher(configs)),
+                        tracker);
+            default:
+                throw new IllegalArgumentException("unknown ir meta reader mode: " + readerMode);
+        }
     }
 
     private static GraphProperties getTestGraph(Configs configs) {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java

@@ -0,0 +1,47 @@
+/*
+ *
+ *  * Copyright 2020 Alibaba Group Holding Limited.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package com.alibaba.graphscope.common.client.channel;
+
+import com.alibaba.graphscope.common.config.Configs;
+import com.alibaba.graphscope.common.config.FrontendConfig;
+import com.google.common.collect.ImmutableList;
+
+import java.net.URI;
+import java.util.List;
+
+// read meta service host from config and return it as URI format
+public class MetaServiceChannelFetcher implements ChannelFetcher<URI> {
+    private static final String schema = "http";
+    private final Configs graphConfig;
+
+    public MetaServiceChannelFetcher(Configs graphConfig) {
+        this.graphConfig = graphConfig;
+    }
+
+    @Override
+    public List<URI> fetch() {
+        String host = FrontendConfig.IR_META_SERVICE_HOST.get(graphConfig);
+        return ImmutableList.of(URI.create(schema + "://" + host));
+    }
+
+    @Override
+    public Type getType() {
+        return Type.HTTP;
+    }
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java

@@ -60,4 +60,16 @@
 
     public static final Config<Integer> PER_QUERY_STREAM_BUFFER_MAX_CAPACITY =
             Config.intConfig("per.query.stream.buffer.max.capacity", 256);
+
+    public static final Config<Long> IR_META_FETCH_INTERVAL_MS =
+            Config.longConfig("ir.meta.fetch.interval.ms", 1000);
+
+    public static final Config<Long> IR_STATISTICS_FETCH_INTERVAL_MS =
+            Config.longConfig("ir.statistics.fetch.interval.ms", 24 * 3600 * 1000l);
+
+    public static Config<String> IR_META_READER_MODE =
+            Config.stringConfig("ir.meta.reader.mode", "local");
+
+    public static Config<String> IR_META_SERVICE_HOST =
+            Config.stringConfig("ir.meta.service.host", "localhost:8080");
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/PlannerConfig.java

@@ -9,7 +9,9 @@ public class PlannerConfig {
     public static final Config<String> GRAPH_PLANNER_OPT =
             Config.stringConfig("graph.planner.opt", "RBO");
     public static final Config<String> GRAPH_PLANNER_RULES =
-            Config.stringConfig("graph.planner.rules", "");
+            Config.stringConfig(
+                    "graph.planner.rules",
+                    "FilterIntoJoinRule,FilterMatchRule,ExtendIntersectRule,ExpandGetVFusionRule");
     public static final Config<Integer> GRAPH_PLANNER_CBO_GLOGUE_SIZE =
             Config.intConfig("graph.planner.cbo.glogue.size", 3);
     public static final Config<Integer> JOIN_MIN_PATTERN_SIZE =