Skip to content

Commit

Permalink
Merge pull request #1380 from facchettos/k0s-in-syncer
Browse files Browse the repository at this point in the history
k0s is now run from the syncer container
  • Loading branch information
FabianKramm authored Nov 28, 2023
2 parents ecb9c49 + 17ec762 commit d02b61e
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 49 deletions.
79 changes: 44 additions & 35 deletions charts/k0s/templates/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ spec:
emptyDir: {}
- name: run-k0s
emptyDir: {}
- name: k0s-binary
emptyDir: {}
- name: k0s-config
secret:
secretName: vc-{{ .Release.Name }}-config
Expand Down Expand Up @@ -111,50 +113,23 @@ spec:
{{ toYaml . | indent 8 }}
{{- end }}
{{- end }}
containers:
{{- if not .Values.vcluster.disabled }}
initContainers:
- image: {{ .Values.defaultImageRegistry }}{{ .Values.vcluster.image }}
name: vcluster
command:
{{- range $f := .Values.vcluster.command }}
- {{ $f | quote }}
{{- end }}
- /bin/sh
args:
{{- range $f := .Values.vcluster.baseArgs }}
- {{ $f | quote }}
{{- end }}
- --status-socket=/run/k0s/status.sock
{{- if not .Values.sync.nodes.enableScheduler }}
- --disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config
{{- else }}
- --disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config
{{- end }}
{{- range $f := .Values.vcluster.extraArgs }}
- {{ $f | quote }}
{{- end }}
env:
{{- if .Values.vcluster.env }}
{{ toYaml .Values.vcluster.env | indent 10 }}
{{- end }}
- name: ETCD_UNSUPPORTED_ARCH
value: arm64
- name: CONFIG_READY
valueFrom:
secretKeyRef:
name: "vc-{{ .Release.Name }}-config"
key: CONFIG_READY
- -c
- "cp /usr/local/bin/k0s /k0s-binary/k0s"
{{- if .Values.vcluster.imagePullPolicy }}
imagePullPolicy: {{ .Values.vcluster.imagePullPolicy }}
{{- end }}
securityContext:
{{ toYaml .Values.securityContext | indent 10 }}
volumeMounts:
- name: run-k0s
mountPath: /run/k0s
{{ toYaml .Values.vcluster.volumeMounts | indent 10 }}
resources:
{{ toYaml .Values.vcluster.resources | indent 10 }}
{{- end }}
- name: k0s-binary
mountPath: /k0s-binary
containers:
{{- if not .Values.syncer.disabled }}
- name: syncer
{{- if .Values.syncer.image }}
Expand Down Expand Up @@ -276,6 +251,30 @@ spec:
fieldRef:
fieldPath: spec.nodeName
{{- end }}
{{- if .Values.vcluster.env }}
{{ toYaml .Values.vcluster.env | indent 10 }}
{{- end }}
- name: ETCD_UNSUPPORTED_ARCH
value: arm64
- name: VCLUSTER_COMMAND
value: |-
command:
{{- range $f := .Values.vcluster.command }}
- {{ $f | quote }}
{{- end }}
args:
{{- range $f := .Values.vcluster.baseArgs }}
- {{ $f | quote }}
{{- end }}
- --status-socket=/run/k0s/status.sock
{{- if not .Values.sync.nodes.enableScheduler }}
- --disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config
{{- else }}
- --disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config
{{- end }}
{{- range $f := .Values.vcluster.extraArgs }}
- {{ $f | quote }}
{{- end }}
{{- if .Values.syncer.env }}
{{ toYaml .Values.syncer.env | indent 10 }}
{{- end }}
Expand All @@ -289,14 +288,24 @@ spec:
volumeMounts:
- name: helm-cache
mountPath: /.cache/helm
- name: k0s-binary
mountPath: /k0s-binary
- name: k0s-config
mountPath: /etc/k0s
- mountPath: /data
name: data
- name: run-k0s
mountPath: /run/k0s
- name: tmp
mountPath: /tmp
{{- if .Values.coredns.enabled }}
- name: coredns
mountPath: /manifests/coredns
readOnly: true
{{- end }}
{{ toYaml .Values.syncer.volumeMounts | indent 10 }}
{{- if .Values.vcluster.volumeMounts }}
{{ toYaml .Values.vcluster.volumeMounts | indent 10 }}
{{- end }}
{{- if .Values.syncer.extraVolumeMounts }}
{{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }}
{{- end }}
Expand Down
11 changes: 1 addition & 10 deletions charts/k0s/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,6 @@ syncer:
enabled: true
readinessProbe:
enabled: true
volumeMounts:
- mountPath: /data
name: data
readOnly: true
extraVolumeMounts: []
resources:
limits:
Expand All @@ -163,18 +159,13 @@ vcluster:
image: k0sproject/k0s:v1.28.2-k0s.0
imagePullPolicy: ""
command:
- k0s
- /k0s-binary/k0s
baseArgs:
- controller
- --config=/etc/k0s/config.yaml
- --data-dir=/data/k0s
# Extra arguments for k0s.
extraArgs: []
volumeMounts:
- mountPath: /data
name: data
- mountPath: /etc/k0s
name: k0s-config
env: []
resources:
limits:
Expand Down
73 changes: 73 additions & 0 deletions pkg/k0s/k0s.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package k0s

import (
"context"
"fmt"
"os"
"os/exec"
"strings"

"github.com/ghodss/yaml"
"github.com/loft-sh/log/scanner"
"github.com/loft-sh/vcluster/pkg/util/loghelper"
"k8s.io/klog/v2"
)

const VClusterCommandEnv = "VCLUSTER_COMMAND"

type k0sCommand struct {
Command []string `json:"command,omitempty"`
Args []string `json:"args,omitempty"`
}

func StartK0S(ctx context.Context) error {
reader, writer, err := os.Pipe()
if err != nil {
return err
}
defer writer.Close()

command := &k0sCommand{}
err = yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command)
if err != nil {
return fmt.Errorf("parsing k0s command %s: %w", os.Getenv(VClusterCommandEnv), err)
}

args := append(command.Command, command.Args...)

// start func
done := make(chan struct{})
go func() {
defer close(done)

// make sure we scan the output correctly
scan := scanner.NewScanner(reader)
for scan.Scan() {
line := scan.Text()
if len(line) == 0 {
continue
}

// print to our logs
args := []interface{}{"component", "k0s"}
loghelper.PrintKlogLine(line, args)
}
}()

// start the command
klog.InfoS("Starting k0s", "args", strings.Join(args, " "))
cmd := exec.CommandContext(ctx, args[0], args[1:]...)
cmd.Stdout = writer
cmd.Stderr = writer
err = cmd.Run()

// make sure we wait for scanner to be done
_ = writer.Close()
<-done

// regular stop case
if err != nil && err.Error() != "signal: killed" {
return err
}
return nil
}
3 changes: 2 additions & 1 deletion pkg/k3s/k3s.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/ghodss/yaml"
"github.com/loft-sh/log/scanner"
"github.com/loft-sh/vcluster/pkg/util/loghelper"
"github.com/loft-sh/vcluster/pkg/util/random"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -62,7 +63,7 @@ func StartK3S(ctx context.Context, serviceCIDR, k3sToken string) error {

// print to our logs
args := []interface{}{"component", "k3s"}
PrintK3sLine(line, args)
loghelper.PrintKlogLine(line, args)
}
}()

Expand Down
13 changes: 12 additions & 1 deletion pkg/setup/initialize.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

"github.com/loft-sh/vcluster/pkg/certs"
"github.com/loft-sh/vcluster/pkg/k0s"
"github.com/loft-sh/vcluster/pkg/k3s"
"github.com/loft-sh/vcluster/pkg/setup/options"
"github.com/loft-sh/vcluster/pkg/specialservices"
Expand Down Expand Up @@ -100,7 +101,17 @@ func initialize(
}

// check if k3s
if !isK0s && certificatesDir != "/pki" {
if isK0s {
// start k0s
go func() {
// we need to run this with the parent ctx as otherwise this context will be cancelled by the wait
// loop in Initialize
err := k0s.StartK0S(parentCtx)
if err != nil {
klog.Fatalf("Error running k0s: %v", err)
}
}()
} else if certificatesDir != "/pki" {
// its k3s, let's create the token secret
k3sToken, err := k3s.EnsureK3SToken(ctx, currentNamespaceClient, currentNamespace, vClusterName)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions pkg/k3s/parse.go → pkg/util/loghelper/klog.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package k3s
package loghelper

import (
"regexp"
Expand All @@ -12,7 +12,7 @@ var klogRegEx1 = regexp.MustCompile(`^[A-Z][0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\.
var structuredComponent = regexp.MustCompile(`^([a-zA-Z\-_]+)=`)

// https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md
func PrintK3sLine(line string, args []interface{}) {
func PrintKlogLine(line string, args []interface{}) {
if klogRegEx1.MatchString(line) {
matches := klogRegEx1.FindStringSubmatch(line)
args = append(args, "location", matches[1])
Expand Down

0 comments on commit d02b61e

Please sign in to comment.