Support Helm deployment

Signed-off-by: Lei Wang <[email protected]>

.dockerignore

@@ -140,3 +140,9 @@ tmp/
 
 apps/networkx/client/python/dist/
 apps/networkx/python_bindings/build/
+
+charts/gart/charts/
+
+k8s/debezium-config.yaml
+k8s/gart-config.yaml
+k8s/writer-deployment.yaml

.gitignore

@@ -139,5 +139,11 @@ tmp/
 apps/networkx/client/python/dist/
 apps/networkx/python_bindings/build/
 
+charts/gart/charts/
+
+k8s/debezium-config.yaml
+k8s/gart-config.yaml
+k8s/writer-deployment.yaml
+
 *.zip
 _deps/

Dockerfile

@@ -23,20 +23,25 @@ FROM ubuntu:22.04
 ARG build_type=All
 
 RUN apt-get update && apt-get install -y \
-  build-essential \
-  cmake \
-  git \
-  lsb-release \
-  sudo \
-  python3 \
-  python3-pip \
-  vim \
-  wget
+    build-essential \
+    cmake \
+    git \
+    lsb-release \
+    sudo \
+    python3 \
+    python3-pip \
+    vim \
+    wget \
+    netcat-openbsd \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN touch /workspace/env_script.sh
 
 RUN if [ "$build_type" = "All" ]; then \
-  apt-get install -y openssh-server \
-  && mkdir -p /var/run/sshd /workspace \
-  && touch /workspace/env_script.sh \
+  apt-get update && apt-get install -y \
+  openssh-server \
+  && rm -rf /var/lib/apt/lists/* \
+  && mkdir -p /var/run/sshd; \
   && echo "mkdir -p /root/.ssh && ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa" >> /workspace/env_script.sh \
   && echo "cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys" >> /workspace/env_script.sh \
   && echo "service ssh start" >> /workspace/env_script.sh; \
@@ -55,9 +60,6 @@ RUN if [ "$build_type" = "All" ]; then \
 WORKDIR /deps
 RUN /workspace/gart/scripts/install-deps.sh /deps $build_type
 
-# Complete the installation
-RUN rm -rf /var/lib/apt/lists/*
-
 # Find the Kafka directory and write its path to a file
 RUN if [ "$build_type" = "All" ]; then \
   set -eux; \

charts/gart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/gart/Chart.yaml

@@ -0,0 +1,22 @@
+annotations:
+  category: Database
+  licenses: Apache-2.0
+apiVersion: v2
+appVersion: 1.16.0
+name: gart
+description: Graph Analysis on Relational Transactional Datasets
+home: https://github.com/graphscope/gart
+type: application
+version: 0.1.0
+maintainers:
+- name: GraphScope
+  url: https://github.com/graphscope
+
+
+dependencies:
+- name: kafka
+  repository: https://charts.bitnami.com/bitnami
+  version: "*"
+- name: etcd
+  repository: https://charts.bitnami.com/bitnami
+  version: "*"

charts/gart/templates/_helpers.tpl

@@ -0,0 +1,166 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "gart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "gart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "gart.debezium.fullname" -}}
+{{- printf "%s-%s" (include "gart.fullname" .) "debezium" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "gart.converter.fullname" -}}
+{{- printf "%s-%s" (include "gart.fullname" .) "converter" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "gart.writer.fullname" -}}
+{{- printf "%s-%s" (include "gart.fullname" .) "writer" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "gart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "gart.labels" -}}
+helm.sh/chart: {{ include "gart.chart" . }}
+{{ include "gart.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "gart.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "gart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Return the proper gart converter image name
+*/}}
+{{- define "gart.converter.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.converter.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper gart writer image name
+*/}}
+{{- define "gart.writer.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.writer.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper gart debezium image name
+*/}}
+{{- define "gart.debezium.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.debezium.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper gart curl image name
+*/}}
+{{- define "gart.curl.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.curl.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Return the configmap with the gart configuration
+*/}}
+{{- define "gart.configmapName" -}}
+{{- if .Values.existingConfigmap -}}
+    {{- printf "%s" (tpl .Values.existingConfigmap $) -}}
+{{- else -}}
+    {{- printf "%s-%s" (include "gart.fullname" .) "config" | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the configmap with the gart debezium configuration
+*/}}
+{{- define "gart.debezium.configmapName" -}}
+{{- if .Values.debezium.existingConfigmap -}}
+    {{- printf "%s" (tpl .Values.debezium.existingConfigmap $) -}}
+{{- else -}}
+    {{- printf "%s-%s" (include "gart.debezium.fullname" .) "config" | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the job name with the gart debezium 
+*/}}
+{{- define "gart.debezium.jobName" -}}
+{{- if .Values.debezium.job -}}
+    {{- printf "%s" (tpl .Values.debezium.job $) -}}
+{{- else -}}
+    {{- printf "%s-%s" (include "gart.debezium.fullname" .) "job" | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}

charts/gart/templates/configmap.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "gart.configmapName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  DB_HOST: {{ .Values.dataconfig.dbHost | quote }}
+  DB_PORT: {{ .Values.dataconfig.dbPort | quote }}
+  DB_USER: {{ .Values.dataconfig.dbUser | quote }}
+  DB_PASSWORD: {{ .Values.dataconfig.dbPassword | quote }}
+  DB_TYPE: {{ .Values.dataconfig.dbType | quote }}
+  DB_NAME: {{ .Values.dataconfig.dbName | quote }}
+  V6D_SOCKET: {{ .Values.dataconfig.v6dSocket | quote }}
+  V6D_SIZE: {{ .Values.dataconfig.v6dSize | quote }}
+  ETCD_PREFIX: {{ .Values.dataconfig.etcdPrefix | quote }}
+  ENABLE_BULKLOAD: {{ .Values.dataconfig.enableBulkload | quote }}
+  SUBGRAPH_NUM: {{ .Values.dataconfig.subgraphNum | quote }}

charts/gart/templates/converter/deployment.yaml

@@ -0,0 +1,57 @@
+{{- $kafka_service_name := .Values.kafka.fullnameOverride }}
+{{- $kafka_service_port := int .Values.kafka.service.ports.client }}
+{{- $kafka_service := printf "%s:%d" $kafka_service_name $kafka_service_port }}
+
+{{- $etcd_service_name := .Values.etcd.fullnameOverride }}
+{{- $etcd_service_port := int .Values.etcd.containerPorts.client }}
+{{- $etcd_service := printf "%s:%d" $etcd_service_name $etcd_service_port }}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "gart.converter.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: {{ .Values.converter.replicaCount }}
+  selector:
+    matchLabels:
+      app: converter
+  template:
+    metadata:
+      labels:
+        app: converter
+    spec:
+      containers:
+      - name: converter
+        image: {{ include "gart.converter.image" . }}
+        imagePullPolicy: {{ .Values.converter.image.pullPolicy | quote }}
+        command: ["/bin/bash", "-c"]
+        args:
+        - |
+          cd /workspace/gart/build &&
+          until nc -z {{ $kafka_service_name }} {{ $kafka_service_port }}; do echo waiting for kafka; sleep 5; done &&
+          until nc -z {{ $etcd_service_name }} {{ $etcd_service_port }}; do echo waiting for etcd; sleep 5; done && 
+          sleep 60 &&
+          ./gart \
+          --db-host ${DB_HOST} \
+          --db-port ${DB_PORT} \
+          --db-name ${DB_NAME} \
+          --db-type ${DB_TYPE} \
+          --user ${DB_USER} \
+          --password ${DB_PASSWORD} \
+          --v6d-sock ${V6D_SOCKET} \
+          --v6d-size ${V6D_SIZE} \
+          --etcd-endpoint {{ $etcd_service }} \
+          --etcd-prefix ${ETCD_PREFIX} \
+          --kafka-server {{ $kafka_service }} \
+          --subgraph-num ${SUBGRAPH_NUM} \
+          --enable-bulkload ${ENABLE_BULKLOAD} \
+          --rg-from-etcd 1 \
+          --k8s-mode yes \
+          --role converter &&
+          sleep infinity
+        envFrom:
+        - configMapRef:
+            name: {{ include "gart.configmapName" . }}
+
+