From 1963f49a970d8042de1d253ffd70595ed298db1d Mon Sep 17 00:00:00 2001 From: fabiankramm Date: Thu, 25 Nov 2021 17:47:55 +0100 Subject: [PATCH] fix: k8s distro deployment improvements --- .dockerignore | 2 +- .github/workflows/release.yaml | 1 + Dockerfile | 12 +++---- charts/k8s/templates/etcd-service.yaml | 4 +++ .../templates/etcd-statefulset-service.yaml | 4 +++ charts/k8s/templates/etcd-statefulset.yaml | 34 +++++-------------- cmd/vcluster/cmd/certs.go | 2 +- devspace.yaml | 4 +-- devspace_start.sh | 2 +- 9 files changed, 25 insertions(+), 40 deletions(-) diff --git a/.dockerignore b/.dockerignore index e11069aae..0743d2943 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,7 @@ /.devspace /.git /.vscode -/chart +/charts /api /kubeconfig.yaml /cmd/virtualclusterctl diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0e6d6a588..6a25d439f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -89,6 +89,7 @@ jobs: helm repo add chartmuseum $CHART_MUSEUM_URL --username $CHART_MUSEUM_USER --password $CHART_MUSEUM_PASSWORD helm cm-push --force --version="$RELEASE_VERSION" charts/k3s/ chartmuseum helm cm-push --force --version="$RELEASE_VERSION" charts/k0s/ chartmuseum + helm cm-push --force --version="$RELEASE_VERSION" charts/k8s/ chartmuseum env: CHART_MUSEUM_URL: "https://charts.loft.sh/" CHART_MUSEUM_USER: ${{ secrets.CHART_MUSEUM_USER }} diff --git a/Dockerfile b/Dockerfile index 73aeeae8e..fe8b8784b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Build the manager binary -FROM golang:1.16 as builder +FROM golang:1.17 as builder -WORKDIR /vcluster +WORKDIR /vcluster-dev ARG TARGETOS ARG TARGETARCH @@ -31,16 +31,12 @@ ENV DEBUG true RUN mkdir -p /.cache /.config ENV GOCACHE=/.cache ENV GOENV=/.config -# Ensure the default group(0) owns all files and folders in /vcluster and /.cache -# to allow sync to /vcluster with devspace and allow go to write into build cache even when run as non-root -RUN chgrp -R 0 /vcluster /.cache /.config && \ - chmod -R g=u /vcluster /.cache /.config # Set home to "/" in order to for kubectl to automatically pick up vcluster kube config ENV HOME / # Build cmd -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -o vcluster cmd/vcluster/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -o /vcluster cmd/vcluster/main.go ENTRYPOINT ["go", "run", "-mod", "vendor", "cmd/vcluster/main.go"] @@ -50,7 +46,7 @@ FROM alpine # Set root path as working directory WORKDIR / -COPY --from=builder /vcluster/vcluster . +COPY --from=builder /vcluster . COPY manifests/ /manifests/ ENTRYPOINT ["/vcluster", "start"] diff --git a/charts/k8s/templates/etcd-service.yaml b/charts/k8s/templates/etcd-service.yaml index 6d8272f78..650487c6e 100644 --- a/charts/k8s/templates/etcd-service.yaml +++ b/charts/k8s/templates/etcd-service.yaml @@ -16,6 +16,10 @@ spec: port: 2379 targetPort: 2379 protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP selector: app: vcluster-etcd release: {{ .Release.Name }} diff --git a/charts/k8s/templates/etcd-statefulset-service.yaml b/charts/k8s/templates/etcd-statefulset-service.yaml index 984e13bf3..568ed816d 100644 --- a/charts/k8s/templates/etcd-statefulset-service.yaml +++ b/charts/k8s/templates/etcd-statefulset-service.yaml @@ -15,6 +15,10 @@ spec: port: 2379 targetPort: 2379 protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP clusterIP: None selector: app: vcluster-etcd diff --git a/charts/k8s/templates/etcd-statefulset.yaml b/charts/k8s/templates/etcd-statefulset.yaml index ca815a622..4086e5501 100644 --- a/charts/k8s/templates/etcd-statefulset.yaml +++ b/charts/k8s/templates/etcd-statefulset.yaml @@ -69,16 +69,18 @@ spec: image: "{{ .Values.etcd.image }}" command: - etcd - - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2379' - '--cert-file=/run/config/pki/etcd-server.crt' - '--client-cert-auth=true' - '--data-dir=/var/lib/etcd' - - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380' - - '--initial-cluster=$(NAME)=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380' + - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2379' + - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380' + - '--initial-cluster=$(NAME)=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380' + - '--listen-client-urls=https://0.0.0.0:2379' + - '--listen-metrics-urls=http://0.0.0.0:2381' + - '--listen-peer-urls=https://0.0.0.0:2380' + - '--initial-cluster-state=new' + - '--initial-cluster-token={{ .Release.Name }}' - '--key-file=/run/config/pki/etcd-server.key' - - '--listen-client-urls=https://0.0.0.1:2379,https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2379' - - '--listen-metrics-urls=http://0.0.0.1:2381' - - '--listen-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd.{{ .Release.Namespace }}:2380' - '--name=$(NAME)' - '--peer-cert-file=/run/config/pki/etcd-peer.crt' - '--peer-client-cert-auth=true' @@ -89,26 +91,6 @@ spec: {{- range $f := .Values.etcd.extraArgs }} - {{ $f | quote }} {{- end }} - livenessProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 8 - startupProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 24 securityContext: {{ toYaml .Values.etcd.securityContext | indent 10 }} env: diff --git a/cmd/vcluster/cmd/certs.go b/cmd/vcluster/cmd/certs.go index 18c4278fa..1113edc99 100644 --- a/cmd/vcluster/cmd/certs.go +++ b/cmd/vcluster/cmd/certs.go @@ -121,7 +121,7 @@ func ExecuteCerts(options *CertsCmd) error { serverSans := []string{etcdService, etcdService + "." + options.Namespace, etcdService + "." + options.Namespace + ".svc"} for i := 0; i < options.EtcdReplicas; i++ { hostname := etcdService + "-" + strconv.Itoa(i) - serverSans = append(serverSans, hostname, hostname+"."+etcdService, hostname+"."+etcdService+"."+options.Namespace) + serverSans = append(serverSans, hostname, hostname+"."+etcdService+"-headless", hostname+"."+etcdService+"-headless"+"."+options.Namespace) } cfg.ClusterName = options.ClusterName diff --git a/devspace.yaml b/devspace.yaml index 7d977a8bd..fcdf37478 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -40,11 +40,9 @@ deployments: enabled: false image: ${SYNCER_IMAGE} noArgs: true - workingDir: /vcluster + workingDir: /vcluster-dev command: ["sleep"] extraArgs: ["99999999999"] - securityContext: - readOnlyRootFilesystem: false dev: terminal: imageSelector: ${SYNCER_IMAGE} diff --git a/devspace_start.sh b/devspace_start.sh index 57a3e7322..b26fe2619 100755 --- a/devspace_start.sh +++ b/devspace_start.sh @@ -4,7 +4,7 @@ set +e # Continue on errors COLOR_CYAN="\033[0;36m" COLOR_RESET="\033[0m" -RUN_CMD="go run -mod vendor cmd/vcluster/main.go" +RUN_CMD="go run -mod vendor cmd/vcluster/main.go start" DEBUG_CMD="dlv debug ./cmd/vcluster/main.go --listen=0.0.0.0:2345 --api-version=2 --output /tmp/__debug_bin --headless --build-flags=\"-mod=vendor\" -- --lease-duration=99999 --renew-deadline=99998" echo -e "${COLOR_CYAN}