From 99a2d15763404e42ecea84f7539e7a772e016a58 Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 28 Nov 2023 09:17:33 +0100 Subject: [PATCH 1/3] k0s is now run from the syncer container --- charts/k0s/templates/statefulset.yaml | 84 ++++++++++++-------- charts/k0s/values.yaml | 5 +- pkg/k0s/k0s.go | 73 +++++++++++++++++ pkg/k3s/k3s.go | 3 +- pkg/setup/initialize.go | 13 ++- pkg/{k3s/parse.go => util/loghelper/klog.go} | 4 +- 6 files changed, 141 insertions(+), 41 deletions(-) create mode 100644 pkg/k0s/k0s.go rename pkg/{k3s/parse.go => util/loghelper/klog.go} (97%) diff --git a/charts/k0s/templates/statefulset.yaml b/charts/k0s/templates/statefulset.yaml index d2dcd77fa..2492ee49c 100644 --- a/charts/k0s/templates/statefulset.yaml +++ b/charts/k0s/templates/statefulset.yaml @@ -78,6 +78,8 @@ spec: emptyDir: {} - name: run-k0s emptyDir: {} + - name: k0s-binary + emptyDir: {} - name: k0s-config secret: secretName: vc-{{ .Release.Name }}-config @@ -111,50 +113,25 @@ spec: {{ toYaml . | indent 8 }} {{- end }} {{- end }} - containers: - {{- if not .Values.vcluster.disabled }} + initContainers: - image: {{ .Values.defaultImageRegistry }}{{ .Values.vcluster.image }} - name: vcluster + name: init command: - {{- range $f := .Values.vcluster.command }} - - {{ $f | quote }} - {{- end }} + - /bin/sh args: - {{- range $f := .Values.vcluster.baseArgs }} - - {{ $f | quote }} - {{- end }} - - --status-socket=/run/k0s/status.sock - {{- if not .Values.sync.nodes.enableScheduler }} - - --disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config - {{- else }} - - --disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config - {{- end }} - {{- range $f := .Values.vcluster.extraArgs }} - - {{ $f | quote }} - {{- end }} - env: - {{- if .Values.vcluster.env }} -{{ toYaml .Values.vcluster.env | indent 10 }} - {{- end }} - - name: ETCD_UNSUPPORTED_ARCH - value: arm64 - - name: CONFIG_READY - valueFrom: - secretKeyRef: - name: "vc-{{ .Release.Name }}-config" - key: CONFIG_READY + - -c + - "cp /usr/local/bin/k0s /k0s-binary/k0s" {{- if .Values.vcluster.imagePullPolicy }} imagePullPolicy: {{ .Values.vcluster.imagePullPolicy }} {{- end }} securityContext: {{ toYaml .Values.securityContext | indent 10 }} volumeMounts: - - name: run-k0s - mountPath: /run/k0s -{{ toYaml .Values.vcluster.volumeMounts | indent 10 }} - resources: -{{ toYaml .Values.vcluster.resources | indent 10 }} - {{- end }} + - name: k0s-binary + mountPath: /k0s-binary + - name: k0s-config + mountPath: /k0s-config + containers: {{- if not .Values.syncer.disabled }} - name: syncer {{- if .Values.syncer.image }} @@ -276,6 +253,35 @@ spec: fieldRef: fieldPath: spec.nodeName {{- end }} + {{- if .Values.vcluster.env }} +{{ toYaml .Values.vcluster.env | indent 10 }} + {{- end }} + - name: ETCD_UNSUPPORTED_ARCH + value: arm64 + - name: CONFIG_READY + valueFrom: + secretKeyRef: + name: "vc-{{ .Release.Name }}-config" + key: CONFIG_READY + - name: VCLUSTER_COMMAND + value: |- + command: + {{- range $f := .Values.vcluster.command }} + - {{ $f | quote }} + {{- end }} + args: + {{- range $f := .Values.vcluster.baseArgs }} + - {{ $f | quote }} + {{- end }} + - --status-socket=/run/k0s/status.sock + {{- if not .Values.sync.nodes.enableScheduler }} + - --disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config + {{- else }} + - --disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config + {{- end }} + {{- range $f := .Values.vcluster.extraArgs }} + - {{ $f | quote }} + {{- end }} {{- if .Values.syncer.env }} {{ toYaml .Values.syncer.env | indent 10 }} {{- end }} @@ -289,6 +295,10 @@ spec: volumeMounts: - name: helm-cache mountPath: /.cache/helm + - name: k0s-binary + mountPath: /k0s-binary + - name: run-k0s + mountPath: /run/k0s - name: tmp mountPath: /tmp {{- if .Values.coredns.enabled }} @@ -359,6 +369,10 @@ spec: volumeDevices: {{ toYaml $container.volumeDevices | indent 10 }} volumeMounts: + - name: run-k0s + mountPath: /run/k0s + - name: k0s-binary + mountPath: /k0s-binary {{ toYaml $container.volumeMounts | indent 10 }} {{- if $container.resources }} resources: diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml index 8eae8382c..78d390323 100644 --- a/charts/k0s/values.yaml +++ b/charts/k0s/values.yaml @@ -146,7 +146,8 @@ syncer: volumeMounts: - mountPath: /data name: data - readOnly: true + - mountPath: /etc/k0s + name: k0s-config extraVolumeMounts: [] resources: limits: @@ -163,7 +164,7 @@ vcluster: image: k0sproject/k0s:v1.28.2-k0s.0 imagePullPolicy: "" command: - - k0s + - /k0s-binary/k0s baseArgs: - controller - --config=/etc/k0s/config.yaml diff --git a/pkg/k0s/k0s.go b/pkg/k0s/k0s.go new file mode 100644 index 000000000..2fc85c7c1 --- /dev/null +++ b/pkg/k0s/k0s.go @@ -0,0 +1,73 @@ +package k0s + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/ghodss/yaml" + "github.com/loft-sh/log/scanner" + "github.com/loft-sh/vcluster/pkg/util/loghelper" + "k8s.io/klog/v2" +) + +const VClusterCommandEnv = "VCLUSTER_COMMAND" + +type k0sCommand struct { + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` +} + +func StartK0S(ctx context.Context) error { + reader, writer, err := os.Pipe() + if err != nil { + return err + } + defer writer.Close() + + command := &k0sCommand{} + err = yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command) + if err != nil { + return fmt.Errorf("parsing k0s command %s: %w", os.Getenv(VClusterCommandEnv), err) + } + + args := append(command.Command, command.Args...) + + // start func + done := make(chan struct{}) + go func() { + defer close(done) + + // make sure we scan the output correctly + scan := scanner.NewScanner(reader) + for scan.Scan() { + line := scan.Text() + if len(line) == 0 { + continue + } + + // print to our logs + args := []interface{}{"component", "k0s"} + loghelper.PrintKlogLine(line, args) + } + }() + + // start the command + klog.InfoS("Starting k0s", "args", strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + cmd.Stdout = writer + cmd.Stderr = writer + err = cmd.Run() + + // make sure we wait for scanner to be done + _ = writer.Close() + <-done + + // regular stop case + if err != nil && err.Error() != "signal: killed" { + return err + } + return nil +} diff --git a/pkg/k3s/k3s.go b/pkg/k3s/k3s.go index 3faec0b77..283d43db5 100644 --- a/pkg/k3s/k3s.go +++ b/pkg/k3s/k3s.go @@ -9,6 +9,7 @@ import ( "github.com/ghodss/yaml" "github.com/loft-sh/log/scanner" + "github.com/loft-sh/vcluster/pkg/util/loghelper" "github.com/loft-sh/vcluster/pkg/util/random" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -62,7 +63,7 @@ func StartK3S(ctx context.Context, serviceCIDR, k3sToken string) error { // print to our logs args := []interface{}{"component", "k3s"} - PrintK3sLine(line, args) + loghelper.PrintKlogLine(line, args) } }() diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go index 4786c1646..662c2349a 100644 --- a/pkg/setup/initialize.go +++ b/pkg/setup/initialize.go @@ -8,6 +8,7 @@ import ( "time" "github.com/loft-sh/vcluster/pkg/certs" + "github.com/loft-sh/vcluster/pkg/k0s" "github.com/loft-sh/vcluster/pkg/k3s" "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/specialservices" @@ -100,7 +101,17 @@ func initialize( } // check if k3s - if !isK0s && certificatesDir != "/pki" { + if isK0s { + // start k0s + go func() { + // we need to run this with the parent ctx as otherwise this context will be cancelled by the wait + // loop in Initialize + err := k0s.StartK0S(parentCtx) + if err != nil { + klog.Fatalf("Error running k0s: %v", err) + } + }() + } else if certificatesDir != "/pki" { // its k3s, let's create the token secret k3sToken, err := k3s.EnsureK3SToken(ctx, currentNamespaceClient, currentNamespace, vClusterName) if err != nil { diff --git a/pkg/k3s/parse.go b/pkg/util/loghelper/klog.go similarity index 97% rename from pkg/k3s/parse.go rename to pkg/util/loghelper/klog.go index f9ee90947..2402c4c2b 100644 --- a/pkg/k3s/parse.go +++ b/pkg/util/loghelper/klog.go @@ -1,4 +1,4 @@ -package k3s +package loghelper import ( "regexp" @@ -12,7 +12,7 @@ var klogRegEx1 = regexp.MustCompile(`^[A-Z][0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\. var structuredComponent = regexp.MustCompile(`^([a-zA-Z\-_]+)=`) // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md -func PrintK3sLine(line string, args []interface{}) { +func PrintKlogLine(line string, args []interface{}) { if klogRegEx1.MatchString(line) { matches := klogRegEx1.FindStringSubmatch(line) args = append(args, "location", matches[1]) From 8ebf06b9c642d86446b69c9bd3cb8135692c1deb Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 28 Nov 2023 09:51:17 +0100 Subject: [PATCH 2/3] adjusted the yaml files --- charts/k0s/templates/statefulset.yaml | 21 ++++++++------------- charts/k0s/values.yaml | 5 ----- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/charts/k0s/templates/statefulset.yaml b/charts/k0s/templates/statefulset.yaml index 2492ee49c..01ea6cfe2 100644 --- a/charts/k0s/templates/statefulset.yaml +++ b/charts/k0s/templates/statefulset.yaml @@ -115,7 +115,7 @@ spec: {{- end }} initContainers: - image: {{ .Values.defaultImageRegistry }}{{ .Values.vcluster.image }} - name: init + name: vcluster command: - /bin/sh args: @@ -129,8 +129,6 @@ spec: volumeMounts: - name: k0s-binary mountPath: /k0s-binary - - name: k0s-config - mountPath: /k0s-config containers: {{- if not .Values.syncer.disabled }} - name: syncer @@ -258,11 +256,6 @@ spec: {{- end }} - name: ETCD_UNSUPPORTED_ARCH value: arm64 - - name: CONFIG_READY - valueFrom: - secretKeyRef: - name: "vc-{{ .Release.Name }}-config" - key: CONFIG_READY - name: VCLUSTER_COMMAND value: |- command: @@ -297,6 +290,10 @@ spec: mountPath: /.cache/helm - name: k0s-binary mountPath: /k0s-binary + - name: k0s-config + mountPath: /etc/k0s + - mountPath: /data + name: data - name: run-k0s mountPath: /run/k0s - name: tmp @@ -306,7 +303,9 @@ spec: mountPath: /manifests/coredns readOnly: true {{- end }} -{{ toYaml .Values.syncer.volumeMounts | indent 10 }} + {{- if .Values.vcluster.volumeMounts }} +{{ toYaml .Values.vcluster.volumeMounts | indent 10 }} + {{- end }} {{- if .Values.syncer.extraVolumeMounts }} {{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }} {{- end }} @@ -369,10 +368,6 @@ spec: volumeDevices: {{ toYaml $container.volumeDevices | indent 10 }} volumeMounts: - - name: run-k0s - mountPath: /run/k0s - - name: k0s-binary - mountPath: /k0s-binary {{ toYaml $container.volumeMounts | indent 10 }} {{- if $container.resources }} resources: diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml index 78d390323..c27b37b9d 100644 --- a/charts/k0s/values.yaml +++ b/charts/k0s/values.yaml @@ -143,11 +143,6 @@ syncer: enabled: true readinessProbe: enabled: true - volumeMounts: - - mountPath: /data - name: data - - mountPath: /etc/k0s - name: k0s-config extraVolumeMounts: [] resources: limits: From 17ec762808d4f907151568aa59c110ca65426010 Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 28 Nov 2023 10:44:35 +0100 Subject: [PATCH 3/3] removed volume mounts --- charts/k0s/values.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml index c27b37b9d..c36645b0c 100644 --- a/charts/k0s/values.yaml +++ b/charts/k0s/values.yaml @@ -166,11 +166,6 @@ vcluster: - --data-dir=/data/k0s # Extra arguments for k0s. extraArgs: [] - volumeMounts: - - mountPath: /data - name: data - - mountPath: /etc/k0s - name: k0s-config env: [] resources: limits: