fix: k8s distro deployment improvements

.dockerignore

@@ -2,7 +2,7 @@
 /.devspace
 /.git
 /.vscode
-/chart
+/charts
 /api
 /kubeconfig.yaml
 /cmd/virtualclusterctl

.github/workflows/release.yaml

@@ -89,6 +89,7 @@ jobs:
           helm repo add chartmuseum $CHART_MUSEUM_URL --username $CHART_MUSEUM_USER --password $CHART_MUSEUM_PASSWORD
           helm cm-push --force --version="$RELEASE_VERSION" charts/k3s/ chartmuseum
           helm cm-push --force --version="$RELEASE_VERSION" charts/k0s/ chartmuseum
+          helm cm-push --force --version="$RELEASE_VERSION" charts/k8s/ chartmuseum
         env:
           CHART_MUSEUM_URL: "https://charts.loft.sh/"
           CHART_MUSEUM_USER: ${{ secrets.CHART_MUSEUM_USER }}

Dockerfile

@@ -1,7 +1,7 @@
 # Build the manager binary
-FROM golang:1.16 as builder
+FROM golang:1.17 as builder
 
-WORKDIR /vcluster
+WORKDIR /vcluster-dev
 ARG TARGETOS
 ARG TARGETARCH
 
@@ -31,16 +31,12 @@ ENV DEBUG true
 RUN mkdir -p /.cache /.config
 ENV GOCACHE=/.cache
 ENV GOENV=/.config
-# Ensure the default group(0) owns all files and folders in /vcluster and /.cache 
-# to allow sync to /vcluster with devspace and allow go to write into build cache even when run as non-root
-RUN chgrp -R 0 /vcluster /.cache /.config && \
-    chmod -R g=u /vcluster /.cache /.config
 
 # Set home to "/" in order to for kubectl to automatically pick up vcluster kube config 
 ENV HOME /
 
 # Build cmd
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -o vcluster cmd/vcluster/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -o /vcluster cmd/vcluster/main.go
 
 ENTRYPOINT ["go", "run", "-mod", "vendor", "cmd/vcluster/main.go"]
 
@@ -50,7 +46,7 @@ FROM alpine
 # Set root path as working directory
 WORKDIR /
 
-COPY --from=builder /vcluster/vcluster .
+COPY --from=builder /vcluster .
 COPY manifests/ /manifests/
 
 ENTRYPOINT ["/vcluster", "start"]

charts/k8s/templates/etcd-service.yaml

@@ -16,6 +16,10 @@ spec:
       port: 2379
       targetPort: 2379
       protocol: TCP
+    - name: peer
+      port: 2380
+      targetPort: 2380
+      protocol: TCP
   selector:
     app: vcluster-etcd
     release: {{ .Release.Name }}

charts/k8s/templates/etcd-statefulset-service.yaml

@@ -15,6 +15,10 @@ spec:
       port: 2379
       targetPort: 2379
       protocol: TCP
+    - name: peer
+      port: 2380
+      targetPort: 2380
+      protocol: TCP
   clusterIP: None
   selector:
     app: vcluster-etcd

charts/k8s/templates/etcd-statefulset.yaml

@@ -69,16 +69,18 @@ spec:
         image: "{{ .Values.etcd.image }}"
         command:
           - etcd
-          - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2379'
           - '--cert-file=/run/config/pki/etcd-server.crt'
           - '--client-cert-auth=true'
           - '--data-dir=/var/lib/etcd'
-          - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380'
-          - '--initial-cluster=$(NAME)=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380'
+          - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2379'
+          - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380'
+          - '--initial-cluster=$(NAME)=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380'
+          - '--listen-client-urls=https://0.0.0.0:2379'
+          - '--listen-metrics-urls=http://0.0.0.0:2381'
+          - '--listen-peer-urls=https://0.0.0.0:2380'
+          - '--initial-cluster-state=new'
+          - '--initial-cluster-token={{ .Release.Name }}'
           - '--key-file=/run/config/pki/etcd-server.key'
-          - '--listen-client-urls=https://0.0.0.1:2379,https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2379'
-          - '--listen-metrics-urls=http://0.0.0.1:2381'
-          - '--listen-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380'
           - '--name=$(NAME)'
           - '--peer-cert-file=/run/config/pki/etcd-peer.crt'
           - '--peer-client-cert-auth=true'
@@ -89,26 +91,6 @@ spec:
           {{- range $f := .Values.etcd.extraArgs }}
           - {{ $f | quote }}
           {{- end }}
-        livenessProbe:
-          httpGet:
-            path: /health
-            port: 2381
-            scheme: HTTP
-          initialDelaySeconds: 10
-          timeoutSeconds: 15
-          periodSeconds: 10
-          successThreshold: 1
-          failureThreshold: 8
-        startupProbe:
-          httpGet:
-            path: /health
-            port: 2381
-            scheme: HTTP
-          initialDelaySeconds: 10
-          timeoutSeconds: 15
-          periodSeconds: 10
-          successThreshold: 1
-          failureThreshold: 24
         securityContext:
 {{ toYaml .Values.etcd.securityContext | indent 10 }}
         env:

cmd/vcluster/cmd/certs.go

@@ -121,7 +121,7 @@ func ExecuteCerts(options *CertsCmd) error {
 	serverSans := []string{etcdService, etcdService + "." + options.Namespace, etcdService + "." + options.Namespace + ".svc"}
 	for i := 0; i < options.EtcdReplicas; i++ {
 		hostname := etcdService + "-" + strconv.Itoa(i)
-		serverSans = append(serverSans, hostname, hostname+"."+etcdService, hostname+"."+etcdService+"."+options.Namespace)
+		serverSans = append(serverSans, hostname, hostname+"."+etcdService+"-headless", hostname+"."+etcdService+"-headless"+"."+options.Namespace)
 	}
 
 	cfg.ClusterName = options.ClusterName

devspace.yaml

@@ -40,11 +40,9 @@ deployments:
             enabled: false
           image: ${SYNCER_IMAGE}
           noArgs: true
-          workingDir: /vcluster
+          workingDir: /vcluster-dev
           command: ["sleep"]
           extraArgs: ["99999999999"]
-        securityContext:
-          readOnlyRootFilesystem: false
 dev:
   terminal:
     imageSelector: ${SYNCER_IMAGE}

devspace_start.sh

@@ -4,7 +4,7 @@ set +e  # Continue on errors
 COLOR_CYAN="\033[0;36m"
 COLOR_RESET="\033[0m"
 
-RUN_CMD="go run -mod vendor cmd/vcluster/main.go"
+RUN_CMD="go run -mod vendor cmd/vcluster/main.go start"
 DEBUG_CMD="dlv debug ./cmd/vcluster/main.go --listen=0.0.0.0:2345 --api-version=2 --output /tmp/__debug_bin --headless --build-flags=\"-mod=vendor\" -- --lease-duration=99999 --renew-deadline=99998"
 
 echo -e "${COLOR_CYAN}