Skip to content

Commit

Permalink
node join commands should be generated properly (#4234)
Browse files Browse the repository at this point in the history 
* first prototype

* include node role, debug logging

* change name of service

* encode cert

* print the old version too for comparison

* use https

* cleanup

* use correct user for controller joins

* wat

* Revert "cleanup"

This reverts commit 55776de.

* more logging

* f

* join controllers with port 9443

* cleanup

* supporess warnings to avoid panic message on kots startup
  • Loading branch information
@laverya
laverya authored Dec 20, 2023
1 parent ec37bb7 commit 4997f6f
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 156 deletions.
206 changes: 62 additions & 144 deletions pkg/embeddedcluster/node_join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ package embeddedcluster

import (
"context"
"encoding/base64"
"fmt"
"strings"
"sync"
"time"

"github.com/replicatedhq/kots/pkg/embeddedcluster/types"
"github.com/replicatedhq/kots/pkg/k8sutil"
"github.com/replicatedhq/kots/pkg/util"
corev1 "k8s.io/api/core/v1"
kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
Expand All @@ -24,6 +24,25 @@ type joinTokenEntry struct {
var joinTokenMapMut = sync.Mutex{}
var joinTokenMap = map[string]*joinTokenEntry{}

const k0sTokenTemplate = `apiVersion: v1
clusters:
- cluster:
certificate-authority-data: %s
server: https://%s:%d
name: k0s
contexts:
- context:
cluster: k0s
user: %s
name: k0s
current-context: k0s
kind: Config
users:
- name: %s
user:
token: %s
`

// GenerateAddNodeToken will generate the embedded cluster node add command for a node with the specified roles
// join commands will last for 24 hours, and will be cached for 1 hour after first generation
func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
Expand All @@ -44,9 +63,9 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node
return joinToken.Token, nil
}

newToken, err := runAddNodeCommandPod(ctx, client, nodeRole)
newToken, err := makeK0sToken(ctx, client, nodeRole)
if err != nil {
return "", fmt.Errorf("failed to run add node command pod: %w", err)
return "", fmt.Errorf("failed to generate k0s token: %w", err)
}

now := time.Now()
Expand All @@ -56,161 +75,60 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node
return newToken, nil
}

// run a pod that will generate the add node token
func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
podName := "k0s-token-generator-"
suffix := strings.Replace(nodeRole, "+", "-", -1)
podName += suffix

// cleanup the pod if it already exists
err := client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{})
func makeK0sToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
rawToken, err := k8sutil.GenerateK0sBootstrapToken(client, time.Hour, nodeRole)
if err != nil {
if !kuberneteserrors.IsNotFound(err) {
return "", fmt.Errorf("failed to delete pod: %w", err)
}
return "", fmt.Errorf("failed to generate bootstrap token: %w", err)
}

hostPathFile := corev1.HostPathFile
hostPathDir := corev1.HostPathDirectory
_, err = client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: "kube-system",
Labels: map[string]string{
"replicated.app/embedded-cluster": "true",
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
HostNetwork: true,
Volumes: []corev1.Volume{
{
Name: "bin",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/usr/local/bin/k0s",
Type: &hostPathFile,
},
},
},
{
Name: "lib",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/var/lib/k0s",
Type: &hostPathDir,
},
},
},
{
Name: "etc",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/etc/k0s",
Type: &hostPathDir,
},
},
},
{
Name: "run",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/run/k0s",
Type: &hostPathDir,
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "node.k0sproject.io/role",
Operator: corev1.NodeSelectorOpIn,
Values: []string{
"control-plane",
},
},
},
},
},
},
},
},
Containers: []corev1.Container{
{
Name: "k0s-token-generator",
Image: "ubuntu:jammy", // this will not work on airgap, but it needs to be debian based at the moment
Command: []string{"/mnt/k0s"},
Args: []string{
"token",
"create",
"--expiry",
"12h",
"--role",
nodeRole,
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "bin",
MountPath: "/mnt/k0s",
},
{
Name: "lib",
MountPath: "/var/lib/k0s",
},
{
Name: "etc",
MountPath: "/etc/k0s",
},
{
Name: "run",
MountPath: "/run/k0s",
},
},
},
},
},
}, metav1.CreateOptions{})
cert, err := k8sutil.GetClusterCaCert(ctx, client)
if err != nil {
return "", fmt.Errorf("failed to create pod: %w", err)
return "", fmt.Errorf("failed to get cluster ca cert: %w", err)
}
cert = base64.StdEncoding.EncodeToString([]byte(cert))

// wait for the pod to complete
for {
pod, err := client.CoreV1().Pods("kube-system").Get(ctx, podName, metav1.GetOptions{})
if err != nil {
return "", fmt.Errorf("failed to get pod: %w", err)
}

if pod.Status.Phase == corev1.PodSucceeded {
break
}

if pod.Status.Phase == corev1.PodFailed {
return "", fmt.Errorf("pod failed")
}
firstPrimary, err := firstPrimaryIpAddress(ctx, client)
if err != nil {
return "", fmt.Errorf("failed to get first primary ip address: %w", err)
}

time.Sleep(time.Second)
userName := "kubelet-bootstrap"
port := 6443
if nodeRole == "controller" {
userName = "controller-bootstrap"
port = 9443
}

// get the logs from the completed pod
podLogs, err := client.CoreV1().Pods("kube-system").GetLogs(podName, &corev1.PodLogOptions{}).DoRaw(ctx)
fullToken := fmt.Sprintf(k0sTokenTemplate, cert, firstPrimary, port, userName, userName, rawToken)
gzipToken, err := util.GzipData([]byte(fullToken))
if err != nil {
return "", fmt.Errorf("failed to get pod logs: %w", err)
return "", fmt.Errorf("failed to gzip token: %w", err)
}
b64Token := base64.StdEncoding.EncodeToString(gzipToken)

return b64Token, nil
}

// delete the completed pod
err = client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{})
func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (string, error) {
nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return "", fmt.Errorf("failed to delete pod: %w", err)
return "", fmt.Errorf("failed to list nodes: %w", err)
}

for _, node := range nodes.Items {
if cp, ok := node.Labels["node-role.kubernetes.io/control-plane"]; !ok || cp != "true" {
continue
}

for _, address := range node.Status.Addresses {
if address.Type == "InternalIP" {
return address.Address, nil
}
}

}

// the logs are just a join token, which needs to be added to other things to get a join command
return string(podLogs), nil
return "", fmt.Errorf("failed to find controller node")
}

// GenerateAddNodeCommand returns the command a user should run to add a node with the provided token
Expand Down
2 changes: 1 addition & 1 deletion pkg/embeddedcluster/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Instal
}
scheme := runtime.NewScheme()
embeddedclusterv1beta1.AddToScheme(scheme)
kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme})
kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme, WarningHandler: kbclient.WarningHandlerOptions{SuppressWarnings: true}})
if err != nil {
return nil, fmt.Errorf("failed to get kubebuilder client: %w", err)
}
Expand Down
56 changes: 45 additions & 11 deletions pkg/k8sutil/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,35 @@ import (
// GenerateBootstrapToken will generate a node join token for kubeadm.
// ttl defines the time to live for this token.
func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (string, error) {
data := map[string][]byte{}
data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.")
for _, usage := range []string{"authentication", "signing"} {
data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
}
data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token")

return generateJoinTokenInternal(client, ttl, data)
}

func GenerateK0sBootstrapToken(client kubernetes.Interface, ttl time.Duration, role string) (string, error) {
data := make(map[string][]byte)

// these 'data' entries are taken from k0s: https://github.com/replicatedhq/k0s/blob/7bc57553ea8ccb6847fdd8249701554ee8be1ab0/pkg/token/manager.go#L69
data["usage-bootstrap-api-auth"] = []byte("true")
if role == "worker" {
data["description"] = []byte("Worker bootstrap token generated by Kotsadm for k0s")
data["usage-bootstrap-authentication"] = []byte("true")
data["usage-bootstrap-api-worker-calls"] = []byte("true")
} else {
data["description"] = []byte("Controller bootstrap token generated by Kotsadm for k0s")
data["usage-bootstrap-authentication"] = []byte("false")
data["usage-bootstrap-signing"] = []byte("false")
data["usage-controller-join"] = []byte("true")
}
return generateJoinTokenInternal(client, ttl, data)
}

func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, data map[string][]byte) (string, error) {
token, err := bootstraputil.GenerateBootstrapToken()
if err != nil {
return "", errors.Wrap(err, "generate kubeadm token")
Expand All @@ -24,21 +53,12 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str
tokenID := substrs[1]
tokenSecret := substrs[2]

data := map[string][]byte{
bootstrapapi.BootstrapTokenIDKey: []byte(tokenID),
bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret),
}
data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.")
data[bootstrapapi.BootstrapTokenIDKey] = []byte(tokenID)
data[bootstrapapi.BootstrapTokenSecretKey] = []byte(tokenSecret)

expirationString := time.Now().Add(ttl).UTC().Format(time.RFC3339)
data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)

for _, usage := range []string{"authentication", "signing"} {
data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
}

data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token")

secretName := fmt.Sprintf("%s%s", bootstrapapi.BootstrapTokenSecretPrefix, tokenID)
bootstrapToken := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -55,3 +75,17 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str

return token, nil
}

func GetClusterCaCert(ctx context.Context, client kubernetes.Interface) (string, error) {
cert, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
if err != nil {
return "", errors.Wrap(err, "failed to get kube-root-ca.crt")
}

caCert, ok := cert.Data["ca.crt"]
if !ok {
return "", fmt.Errorf("ca.crt not found in kube-root-ca.crt, actual data was %v", cert.Data)
}

return caCert, nil
}

0 comments on commit 4997f6f

Please sign in to comment.