merge

Committed-by: xiaolei.zl from Dev container
zhanglei1949 · Nov 4, 2024 · dbde861 · dbde861
2 parents 87e948a + 25fa9e9
commit dbde861
Show file tree

Hide file tree

Showing 34 changed files with 541 additions and 159 deletions.
diff --git a/charts/graphscope-store/templates/configmap.yaml b/charts/graphscope-store/templates/configmap.yaml
@@ -116,6 +116,7 @@ data:
 
     sudo chown -R graphscope:graphscope {{ .Values.storeDataPath }} || true
     sudo chown -R graphscope:graphscope /etc/groot || true
+    sudo chown -R graphscope:graphscope /var/log/graphscope || true
 
     [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
     ordinal=${BASH_REMATCH[1]}

diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml
@@ -135,20 +135,31 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath:  {{ .Values.logPersistence.mountPath }}
       volumes:
         - name: config
           configMap:
             name: {{ include "graphscope-store.configmapName" . }}
             defaultMode: 0755
-  {{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
+        {{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
         - name: meta
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.coordinator.persistence.existingClaim .) }}
-  {{- else if not .Values.coordinator.persistence.enabled }}
+        {{- else if not .Values.coordinator.persistence.enabled }}
         - name: meta
           emptyDir: {}
-  {{- else if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
     - metadata:
         name: meta
         {{- if .Values.persistence.annotations }}
@@ -169,5 +180,27 @@ spec:
         {{- if .Values.coordinator.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
-{{- end -}}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
+{{- end -}}
diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml
@@ -137,9 +137,42 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
       volumes:
         - name: config
           configMap:
             name: {{ include "graphscope-store.configmapName" . }}
             defaultMode: 0755
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
+  volumeClaimTemplates:
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end }}
diff --git a/charts/graphscope-store/templates/onepod/statefulset.yaml b/charts/graphscope-store/templates/onepod/statefulset.yaml
@@ -137,6 +137,8 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
             {{- if .Values.store.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }}
             {{- end }}
@@ -148,15 +150,24 @@ spec:
         {{- if .Values.store.extraVolumes }}
         {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }}
         {{- end }}
-  {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
+        {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
         - name: data
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }}
-  {{- else if not .Values.store.persistence.enabled }}
+        {{- else if not .Values.store.persistence.enabled }}
         - name: data
           emptyDir: {}
-  {{- else if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
     - metadata:
         name: data
         {{- if .Values.persistence.annotations }}
@@ -177,5 +188,27 @@ spec:
         {{- if .Values.store.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end }}
diff --git a/charts/graphscope-store/templates/store/statefulset-backup.yaml b/charts/graphscope-store/templates/store/statefulset-backup.yaml
@@ -143,6 +143,8 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
             {{- if .Values.store.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }}
             {{- end }}
@@ -154,15 +156,24 @@ spec:
         {{- if .Values.store.extraVolumes }}
         {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }}
         {{- end }}
-  {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
+        {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
         - name: data
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }}
-  {{- else if not .Values.store.persistence.enabled }}
+        {{- else if not .Values.store.persistence.enabled }}
         - name: data
           emptyDir: {}
-  {{- else if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
     - metadata:
         name: data
         {{- if .Values.persistence.annotations }}
@@ -183,6 +194,28 @@ spec:
         {{- if .Values.store.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end -}}
 {{- end -}}
diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml
@@ -139,6 +139,8 @@ spec:
             - name: config
               mountPath: /etc/groot/setup.sh
               subPath: setup.sh
+            - name: log
+              mountPath: {{ .Values.logPersistence.mountPath }}
             {{- if .Values.store.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }}
             {{- end }}
@@ -150,15 +152,24 @@ spec:
         {{- if .Values.store.extraVolumes }}
         {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }}
         {{- end }}
-  {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
+        {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
         - name: data
           persistentVolumeClaim:
             claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }}
-  {{- else if not .Values.store.persistence.enabled }}
+        {{- else if not .Values.store.persistence.enabled }}
         - name: data
           emptyDir: {}
-  {{- else if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
+        {{- end }}
+        {{- if not .Values.logPersistence.enabled }}
+        - name: log
+          emptyDir: {}
+        {{- else if .Values.logPersistence.existingClaim }}
+        - name: log
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+        {{- end }}
   volumeClaimTemplates:
+    {{- if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
     - metadata:
         name: data
         {{- if .Values.persistence.annotations }}
@@ -179,5 +190,27 @@ spec:
         {{- if .Values.store.persistence.selector }}
         selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
         {{- end -}}
-  {{- end }}
+    {{- end }}
+    {{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: log
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if .Values.persistence.labels }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+          {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "graphscope-store.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+    {{- end }}
 {{- end -}}
diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml
@@ -595,6 +595,27 @@ queryExecutionTimeoutMs: 600000
 graphPlannerJoinMinPatternSize: 5
 graphPlannerCboGlogueSize: 3
 
+## Log Persistence parameters
+##
+logPersistence:
+  ## Enable persistence using Persistent Volume Claims. If false, use emptyDir
+  enabled: false
+  ## Existing PVC to hold the log data. If not provided, a new PVC will be created.
+  existingClaim: ""
+  ## Persistent Volume storage class
+  storageClass: ""
+  ## Persistent Volume access modes
+  accessModes:
+  - ReadWriteOnce
+  ## Persistent Volume size
+  size: 1Gi
+  ## Persistent Volume annotations
+  annotations: {}
+  ## Persistent Volume labels
+  selector: {}
+  ## mountPath for the Persistent Volume
+  mountPath: "/var/log/graphscope"
+
 ## Key-value pair separated by ;
 ## For example extraConfig="k1=v1;k2=v2"
 extraConfig: ""

diff --git a/...ne/common/src/main/java/com/alibaba/graphscope/groot/common/config/CoordinatorConfig.java b/...ne/common/src/main/java/com/alibaba/graphscope/groot/common/config/CoordinatorConfig.java
@@ -17,7 +17,7 @@
 
 public class CoordinatorConfig {
     public static final Config<Long> SNAPSHOT_INCREASE_INTERVAL_MS =
-            Config.longConfig("snapshot.increase.interval.ms", 1000L);
+            Config.longConfig("snapshot.increase.interval.ms", 2000L);
 
     public static final Config<Long> OFFSETS_PERSIST_INTERVAL_MS =
             Config.longConfig("offsets.persist.interval.ms", 1000L);

diff --git a/...e_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/ExecutionClient.java b/...e_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/ExecutionClient.java
@@ -21,6 +21,7 @@
 import com.alibaba.graphscope.common.client.type.ExecutionResponseListener;
 import com.alibaba.graphscope.common.config.Configs;
 import com.alibaba.graphscope.common.config.QueryTimeoutConfig;
+import com.alibaba.graphscope.gremlin.plugin.QueryLogger;
 
 /**
  * client to submit request to remote engine service
@@ -37,7 +38,8 @@ public ExecutionClient(ChannelFetcher<C> channelFetcher) {
     public abstract void submit(
             ExecutionRequest request,
             ExecutionResponseListener listener,
-            QueryTimeoutConfig timeoutConfig)
+            QueryTimeoutConfig timeoutConfig,
+            QueryLogger queryLogger)
             throws Exception;
 
     public abstract void close() throws Exception;

diff --git a/...gine/compiler/src/main/java/com/alibaba/graphscope/common/client/HttpExecutionClient.java b/...gine/compiler/src/main/java/com/alibaba/graphscope/common/client/HttpExecutionClient.java
@@ -25,6 +25,7 @@
 import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical;
 import com.alibaba.graphscope.gaia.proto.IrResult;
 import com.alibaba.graphscope.gaia.proto.StoredProcedure;
+import com.alibaba.graphscope.gremlin.plugin.QueryLogger;
 import com.alibaba.graphscope.interactive.client.Session;
 import com.alibaba.graphscope.interactive.client.common.Result;
 import com.alibaba.graphscope.interactive.client.impl.DefaultSession;
@@ -56,7 +57,8 @@ public HttpExecutionClient(Configs graphConfig, ChannelFetcher<URI> channelFetch
     public void submit(
             ExecutionRequest request,
             ExecutionResponseListener listener,
-            QueryTimeoutConfig timeoutConfig)
+            QueryTimeoutConfig timeoutConfig,
+            QueryLogger queryLogger)
             throws Exception {
         List<CompletableFuture> responseFutures = Lists.newArrayList();
         for (URI httpURI : channelFetcher.fetch()) {