merge

Committed-by: [email protected] from Dev container

.github/workflows/flex-interactive.yml

@@ -56,7 +56,7 @@ jobs:
         # install gsctl
         python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl
         # launch service: 8080 for coordinator http port; 7687 for cypher port;
-        gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --interactive-config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
+        gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --cypher-port 7688 --interactive-config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
         sleep 20
         # test
         python3 -m pip install --no-cache-dir pytest pytest-cov pytest-timeout pytest-xdist
@@ -66,6 +66,10 @@ jobs:
                           --exitfirst \
                           $(dirname $(python3 -c "import graphscope.gsctl as gsctl; print(gsctl.__file__)"))/tests/test_interactive.py
 
+        # test coordinator
+        res=`curl http://127.0.0.1:8080/api/v1/service`
+        echo $res | grep 7688 || exit 1
+
         # destroy instance
         gsctl instance destroy --type interactive -y
 

.github/workflows/interactive.yml

@@ -24,8 +24,52 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  build-interactive:
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository == 'alibaba/GraphScope' }}
+    container:
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install latest libgrape-lite
+      if: false
+      run: |
+        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
+        cd /tmp/libgrape-lite
+        mkdir -p build && cd build
+        cmake ..
+        make -j$(nproc)
+        make install
+
+    - name: Setup tmate session
+      if: false
+      uses: mxschmitt/action-tmate@v3
+
+    - name: Build Interactive
+      env:
+        GIE_HOME: ${{ github.workspace }}/interactive_engine/
+        HOME: /home/graphscope/
+      run: |
+        . /home/graphscope/.graphscope_env
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
+        cd ${GITHUB_WORKSPACE}/flex
+        mkdir build && cd build
+        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope && sudo make -j$(nproc)
+        # package the build artifacts
+        cd .. && tar -zcf build.tar.gz build
+    
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
+        path: |
+          ${{ github.workspace }}/flex/build.tar.gz
+
+
   test-hqps-engine:
     runs-on: ubuntu-20.04
+    needs: build-interactive
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
       image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -58,23 +102,28 @@ jobs:
         cmake ..
         make -j$(nproc)
         make install
+    
+    - name: Download Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
 
     - name: Setup tmate session
       if: false
       uses: mxschmitt/action-tmate@v3
 
+    - name: Extract build artifacts
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        tar zxf build.tar.gz -C flex && rm build.tar.gz
+
     - name: Build
       env:
         GIE_HOME: ${{ github.workspace }}/interactive_engine/
         HOME: /home/graphscope/
       run: |
         . /home/graphscope/.graphscope_env
         cd ${GITHUB_WORKSPACE}/
-        git submodule update --init
-        cd ${GITHUB_WORKSPACE}/flex
-        mkdir build && cd build
-        cmake .. && sudo make -j$(nproc)
-        sudo make install
 
         # cargo
         . /home/graphscope/.cargo/env
@@ -175,7 +224,7 @@ jobs:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps
-        bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml
+        bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml
 
     - name: Sample Query test
       env:
@@ -400,6 +449,7 @@ jobs:
 
   test-flex:
     runs-on: ubuntu-20.04
+    needs: build-interactive
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
       image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -416,14 +466,15 @@ jobs:
         make -j$(nproc)
         make install
 
-    - name: Build
-      env:
-        HOME: /home/graphscope/
+    - name: Download Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: interactive_build-${{ github.sha }}
+
+    - name: Extract build artifacts
       run: |
-        cd ${GITHUB_WORKSPACE}/flex
-        git submodule update --init
-        mkdir build && cd build
-        cmake .. && sudo make -j$(nproc)
+        cd ${GITHUB_WORKSPACE}
+        tar zxf build.tar.gz -C flex && rm build.tar.gz
 
     - name: Test GRIN on mutable csr 
       run: |
@@ -554,3 +605,15 @@ jobs:
         SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_schema_test.yaml
         BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_import_test.yaml
         GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
+    
+    - name: Test graph loading with different delimiter
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest/
+        FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/modern_graph_tab_delimiter/
+      run: |
+        rm -rf /tmp/csr-data-dir/
+        cd ${GITHUB_WORKSPACE}/flex/build/
+        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
+        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
+        sed -i 's/|/\\t/g' ${BULK_LOAD_FILE}
+        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

charts/graphscope-store/templates/configmap.yaml

@@ -116,6 +116,7 @@ data:
 
     sudo chown -R graphscope:graphscope {{ .Values.storeDataPath }} || true
     sudo chown -R graphscope:graphscope /etc/groot || true
+    sudo chown -R graphscope:graphscope /var/log/graphscope || true
 
     [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
     ordinal=${BASH_REMATCH[1]}
@@ -180,7 +181,12 @@ data:
       export GRPC_PORT=${GROOT_GRPC_PORT}
       export GREMLIN_PORT=${GROOT_GREMLIN_PORT}
       echo "${HOST} ${GRPC_PORT} ${GREMLIN_PORT}"
-      python3 -c 'import base64;import os;from gremlin_python.driver.client import Client;ip=os.getenv("HOST");gremlin_port=os.getenv("GREMLIN_PORT");graph_url=f"ws://{ip}:{gremlin_port}/gremlin";username=os.getenv("GROOT_USERNAME");password=base64.b64decode(os.getenv("GROOT_PASSWORD")).decode("utf-8");client = Client(graph_url, "g", username=username, password=password); ret = client.submit("g.V().limit(1)").all().result(); client.close();' && break
+      cmd="nc -zv ${HOST} ${GREMLIN_PORT}"
+      res=$(eval $cmd 2>&1)
+      if [[ $res == *succeeded* ]]; then
+        # Expected Output is <Connection to localhost (::1) 8182 port [tcp/*] succeeded!>
+        break
+      fi
       sleep 3
     done
 

charts/graphscope-store/templates/coordinator/statefulset.yaml

@@ -135,20 +135,31 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath:  {{ .Values.logPersistence.mountPath }}
       volumes:
         - name: config
           configMap:
             name: {{ include "graphscope-store.configmapName" . }}
             defaultMode: 0755
-  {{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
+        {{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
         - name: meta
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.coordinator.persistence.existingClaim .) }}
-  {{- else if not .Values.coordinator.persistence.enabled }}
+        {{- else if not .Values.coordinator.persistence.enabled }}
         - name: meta
           emptyDir: {}
-  {{- else if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
     - metadata:
         name: meta
         {{- if .Values.persistence.annotations }}
@@ -169,5 +180,27 @@ spec:
         {{- if .Values.coordinator.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
-{{- end -}}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
+{{- end -}}

charts/graphscope-store/templates/frontend/statefulset.yaml

@@ -137,9 +137,42 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
       volumes:
         - name: config
           configMap:
             name: {{ include "graphscope-store.configmapName" . }}
             defaultMode: 0755
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
+  volumeClaimTemplates:
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end }}

charts/graphscope-store/templates/onepod/statefulset.yaml

@@ -137,6 +137,8 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
             {{- if .Values.store.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }}
             {{- end }}
@@ -148,15 +150,24 @@ spec:
         {{- if .Values.store.extraVolumes }}
         {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }}
         {{- end }}
-  {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
+        {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
         - name: data
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }}
-  {{- else if not .Values.store.persistence.enabled }}
+        {{- else if not .Values.store.persistence.enabled }}
         - name: data
           emptyDir: {}
-  {{- else if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
     - metadata:
         name: data
         {{- if .Values.persistence.annotations }}
@@ -177,5 +188,27 @@ spec:
         {{- if .Values.store.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end }}

charts/graphscope-store/templates/portal/statefulset.yaml

@@ -115,6 +115,8 @@ spec:
               value: {{ .Values.portal.runtimePath | quote }}
             - name: STUDIO_WRAPPER_ENDPOINT
               value: {{ .Values.portal.studioWrapperEndpoint | quote }}
+            - name: BASEID
+              value: {{ .Values.portal.baseId | quote }}
             - name: SOLUTION
               value: "GRAPHSCOPE_INSIGHT"
             {{- range $key, $value := .Values.env }}