From 9b268d55c66c862af1ad5458c4adfdfd0ed5d549 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 15:50:31 -0600 Subject: [PATCH 1/6] generate a node join token --- pkg/handlers/helmvm_node_join_command.go | 4 +- pkg/helmvm/node_join.go | 201 ++++++++++++++++++++++- 2 files changed, 201 insertions(+), 4 deletions(-) diff --git a/pkg/handlers/helmvm_node_join_command.go b/pkg/handlers/helmvm_node_join_command.go index 6604b659d9..f300c8bda5 100644 --- a/pkg/handlers/helmvm_node_join_command.go +++ b/pkg/handlers/helmvm_node_join_command.go @@ -22,7 +22,7 @@ func (h *Handler) GenerateHelmVMNodeJoinCommandSecondary(w http.ResponseWriter, return } - command, expiry, err := helmvm.GenerateAddNodeCommand(client, false) + command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, false) if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) @@ -42,7 +42,7 @@ func (h *Handler) GenerateHelmVMNodeJoinCommandPrimary(w http.ResponseWriter, r return } - command, expiry, err := helmvm.GenerateAddNodeCommand(client, true) + command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, true) if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index 6aad6255a9..400c03db56 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -1,12 +1,209 @@ package helmvm import ( + "context" + "fmt" + "sync" "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) +var addNodeMut = sync.Mutex{} +var primaryNodeJoinCommand []string +var primaryNodeJoinCommandCreation *time.Time +var secondaryNodeJoinCommand []string +var secondaryNodeJoinCommandCreation *time.Time + // GenerateAddNodeCommand will generate the HelmVM node add command for a primary or secondary node -func GenerateAddNodeCommand(client kubernetes.Interface, primary bool) ([]string, *time.Time, error) { - return nil, nil, nil +// join commands will last for 24 hours, and will be cached for 1 hour after first generation +func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, primary bool) ([]string, *time.Time, error) { + addNodeMut.Lock() + defer addNodeMut.Unlock() + + if primary { + if primaryNodeJoinCommandCreation != nil && time.Now().Before(primaryNodeJoinCommandCreation.Add(time.Hour)) { + expiry := primaryNodeJoinCommandCreation.Add(time.Hour * 24) + return primaryNodeJoinCommand, &expiry, nil + } + } else { + if secondaryNodeJoinCommandCreation != nil && time.Now().Before(secondaryNodeJoinCommandCreation.Add(time.Hour)) { + expiry := secondaryNodeJoinCommandCreation.Add(time.Hour * 24) + return secondaryNodeJoinCommand, &expiry, nil + } + } + + newCmd, err := runAddNodeCommandPod(ctx, client, primary) + if err != nil { + return nil, nil, fmt.Errorf("failed to run add node command pod: %w", err) + } + + now := time.Now() + if primary { + primaryNodeJoinCommand = newCmd + primaryNodeJoinCommandCreation = &now + } else { + secondaryNodeJoinCommand = newCmd + secondaryNodeJoinCommandCreation = &now + } + + expiry := now.Add(time.Hour * 24) + return newCmd, &expiry, nil +} + +// run a pod that will generate the add node command +func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, primary bool) ([]string, error) { + podName := "k0s-token-generator" + nodeRole := "" + if primary { + podName += "-primary" + nodeRole = "controller" + } else { + podName += "-secondary" + nodeRole = "worker" + } + + // cleanup the pod if it already exists + _ = client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) + + hostPathFile := corev1.HostPathFile + hostPathDir := corev1.HostPathDirectory + _, err := client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: "kube-system", + Labels: map[string]string{ + "replicated.app/embedded-cluster": "true", + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "bin", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/usr/local/bin/k0s", + Type: &hostPathFile, + }, + }, + }, + { + Name: "lib", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/k0s", + Type: &hostPathDir, + }, + }, + }, + { + Name: "etc", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/k0s", + Type: &hostPathDir, + }, + }, + }, + { + Name: "run", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/k0s", + Type: &hostPathDir, + }, + }, + }, + }, + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "node.k0sproject.io/role", + Operator: corev1.NodeSelectorOpIn, + Values: []string{ + "control-plane", + }, + }, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "k0s-token-generator", + Image: "ubuntu:latest", // TODO use the kotsadm image here as we'll know it exists + Command: []string{"/mnt/k0s"}, + Args: []string{ + "token", + "create", + "--expiry", + "12h", + "--role", + nodeRole, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "bin", + MountPath: "/mnt/k0s", + }, + { + Name: "lib", + MountPath: "/var/lib/k0s", + }, + { + Name: "etc", + MountPath: "/etc/k0s", + }, + { + Name: "run", + MountPath: "/run/k0s", + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create pod: %w", err) + } + + // wait for the pod to complete + for { + pod, err := client.CoreV1().Pods("kube-system").Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get pod: %w", err) + } + + if pod.Status.Phase == corev1.PodSucceeded { + break + } + + if pod.Status.Phase == corev1.PodFailed { + return nil, fmt.Errorf("pod failed") + } + + time.Sleep(time.Second) + } + + // get the logs from the completed pod + podLogs, err := client.CoreV1().Pods("kube-system").GetLogs(podName, &corev1.PodLogOptions{}).DoRaw(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get pod logs: %w", err) + } + + // the logs are just a join token, which needs to be added to other things to get a join command + token := string(podLogs) + + // for now, just return the token + + return []string{token}, nil } From 355bfaea0411dbfe4bc9b61bd7a329f40bc38bf1 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 16:10:08 -0600 Subject: [PATCH 2/6] two mutexes, and do not restart successful pods --- pkg/helmvm/node_join.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index 400c03db56..8f53c336d8 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -11,7 +11,8 @@ import ( "k8s.io/client-go/kubernetes" ) -var addNodeMut = sync.Mutex{} +var addPrimaryNodeMut = sync.Mutex{} +var addSecondaryNodeMut = sync.Mutex{} var primaryNodeJoinCommand []string var primaryNodeJoinCommandCreation *time.Time var secondaryNodeJoinCommand []string @@ -20,8 +21,13 @@ var secondaryNodeJoinCommandCreation *time.Time // GenerateAddNodeCommand will generate the HelmVM node add command for a primary or secondary node // join commands will last for 24 hours, and will be cached for 1 hour after first generation func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, primary bool) ([]string, *time.Time, error) { - addNodeMut.Lock() - defer addNodeMut.Unlock() + if primary { + addPrimaryNodeMut.Lock() + defer addPrimaryNodeMut.Unlock() + } else { + addSecondaryNodeMut.Lock() + defer addSecondaryNodeMut.Unlock() + } if primary { if primaryNodeJoinCommandCreation != nil && time.Now().Before(primaryNodeJoinCommandCreation.Add(time.Hour)) { @@ -79,7 +85,8 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim }, }, Spec: corev1.PodSpec{ - HostNetwork: true, + RestartPolicy: corev1.RestartPolicyOnFailure, + HostNetwork: true, Volumes: []corev1.Volume{ { Name: "bin", From 59f465a8d3e442475594d0be7c99f2b9df2821ed Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Oct 2023 17:23:20 -0600 Subject: [PATCH 3/6] generate the full node join command --- pkg/helmvm/jointoken.go | 25 +++++++++++++++ pkg/helmvm/node_join.go | 69 ++++++++++++++++++++++++++++++++++------- pkg/helmvm/util.go | 2 +- 3 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 pkg/helmvm/jointoken.go diff --git a/pkg/helmvm/jointoken.go b/pkg/helmvm/jointoken.go new file mode 100644 index 0000000000..a21fa1fb3a --- /dev/null +++ b/pkg/helmvm/jointoken.go @@ -0,0 +1,25 @@ +package helmvm + +import ( + "encoding/base64" + "encoding/json" + + "github.com/google/uuid" +) + +// joinToken is a struct that holds both the actual token and the cluster id. This is marshaled +// and base64 encoded and used as argument to the join command in the other nodes. +type joinToken struct { + ClusterID uuid.UUID `json:"clusterID"` + Token string `json:"token"` + Role string `json:"role"` +} + +// Encode encodes a JoinToken to base64. +func (j *joinToken) Encode() (string, error) { + b, err := json.Marshal(j) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(b), nil +} diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index 8f53c336d8..eed23cd2df 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -6,7 +6,9 @@ import ( "sync" "time" + "github.com/google/uuid" corev1 "k8s.io/api/core/v1" + kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -41,11 +43,16 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr } } - newCmd, err := runAddNodeCommandPod(ctx, client, primary) + newToken, err := runAddNodeCommandPod(ctx, client, primary) if err != nil { return nil, nil, fmt.Errorf("failed to run add node command pod: %w", err) } + newCmd, err := generateAddNodeCommand(ctx, client, primary, newToken) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate add node command: %w", err) + } + now := time.Now() if primary { primaryNodeJoinCommand = newCmd @@ -59,8 +66,8 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr return newCmd, &expiry, nil } -// run a pod that will generate the add node command -func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, primary bool) ([]string, error) { +// run a pod that will generate the add node token +func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, primary bool) (string, error) { podName := "k0s-token-generator" nodeRole := "" if primary { @@ -72,11 +79,16 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim } // cleanup the pod if it already exists - _ = client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) + err := client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) + if err != nil { + if !kuberneteserrors.IsNotFound(err) { + return "", fmt.Errorf("failed to delete pod: %w", err) + } + } hostPathFile := corev1.HostPathFile hostPathDir := corev1.HostPathDirectory - _, err := client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{ + _, err = client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: "kube-system", @@ -180,14 +192,14 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim }, }, metav1.CreateOptions{}) if err != nil { - return nil, fmt.Errorf("failed to create pod: %w", err) + return "", fmt.Errorf("failed to create pod: %w", err) } // wait for the pod to complete for { pod, err := client.CoreV1().Pods("kube-system").Get(ctx, podName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get pod: %w", err) + return "", fmt.Errorf("failed to get pod: %w", err) } if pod.Status.Phase == corev1.PodSucceeded { @@ -195,7 +207,7 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim } if pod.Status.Phase == corev1.PodFailed { - return nil, fmt.Errorf("pod failed") + return "", fmt.Errorf("pod failed") } time.Sleep(time.Second) @@ -204,13 +216,46 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim // get the logs from the completed pod podLogs, err := client.CoreV1().Pods("kube-system").GetLogs(podName, &corev1.PodLogOptions{}).DoRaw(ctx) if err != nil { - return nil, fmt.Errorf("failed to get pod logs: %w", err) + return "", fmt.Errorf("failed to get pod logs: %w", err) } // the logs are just a join token, which needs to be added to other things to get a join command - token := string(podLogs) + return string(podLogs), nil +} - // for now, just return the token +// generate the add node command from the join token, whether this is a primary node, and info from the embedded-cluster-config configmap +func generateAddNodeCommand(ctx context.Context, client kubernetes.Interface, primary bool, token string) ([]string, error) { + cm, err := ReadConfigMap(client) + if err != nil { + return nil, fmt.Errorf("failed to read configmap: %w", err) + } + + clusterID := cm.Data["embedded-cluster-id"] + binaryName := cm.Data["embedded-binary-name"] + + clusterUUID := uuid.UUID{} + err = clusterUUID.UnmarshalText([]byte(clusterID)) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal cluster id %s: %w", clusterID, err) + } + + nodeRole := "" + if primary { + nodeRole = "controller" + } else { + nodeRole = "worker" + } + + fullToken := joinToken{ + ClusterID: clusterUUID, + Token: token, + Role: nodeRole, + } + + b64token, err := fullToken.Encode() + if err != nil { + return nil, fmt.Errorf("unable to encode token: %w", err) + } - return []string{token}, nil + return []string{binaryName, b64token}, nil } diff --git a/pkg/helmvm/util.go b/pkg/helmvm/util.go index 5dd2cdc11f..ce358abab0 100644 --- a/pkg/helmvm/util.go +++ b/pkg/helmvm/util.go @@ -3,8 +3,8 @@ package helmvm import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" From 668a44807c515932e36555aab5989e4a39d95c81 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Oct 2023 10:05:07 -0600 Subject: [PATCH 4/6] all controllers are also workers --- pkg/helmvm/jointoken.go | 2 +- pkg/helmvm/node_join.go | 35 +++++++++++++---------------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pkg/helmvm/jointoken.go b/pkg/helmvm/jointoken.go index a21fa1fb3a..d723f6f015 100644 --- a/pkg/helmvm/jointoken.go +++ b/pkg/helmvm/jointoken.go @@ -3,7 +3,7 @@ package helmvm import ( "encoding/base64" "encoding/json" - + "github.com/google/uuid" ) diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index eed23cd2df..d2ed53dec4 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -3,6 +3,7 @@ package helmvm import ( "context" "fmt" + "strings" "sync" "time" @@ -31,24 +32,27 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr defer addSecondaryNodeMut.Unlock() } + nodeRole := "" if primary { if primaryNodeJoinCommandCreation != nil && time.Now().Before(primaryNodeJoinCommandCreation.Add(time.Hour)) { expiry := primaryNodeJoinCommandCreation.Add(time.Hour * 24) return primaryNodeJoinCommand, &expiry, nil } + nodeRole = "controller+worker" } else { if secondaryNodeJoinCommandCreation != nil && time.Now().Before(secondaryNodeJoinCommandCreation.Add(time.Hour)) { expiry := secondaryNodeJoinCommandCreation.Add(time.Hour * 24) return secondaryNodeJoinCommand, &expiry, nil } + nodeRole = "worker" } - newToken, err := runAddNodeCommandPod(ctx, client, primary) + newToken, err := runAddNodeCommandPod(ctx, client, nodeRole) if err != nil { return nil, nil, fmt.Errorf("failed to run add node command pod: %w", err) } - newCmd, err := generateAddNodeCommand(ctx, client, primary, newToken) + newCmd, err := generateAddNodeCommand(ctx, client, nodeRole, newToken) if err != nil { return nil, nil, fmt.Errorf("failed to generate add node command: %w", err) } @@ -67,16 +71,10 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr } // run a pod that will generate the add node token -func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, primary bool) (string, error) { - podName := "k0s-token-generator" - nodeRole := "" - if primary { - podName += "-primary" - nodeRole = "controller" - } else { - podName += "-secondary" - nodeRole = "worker" - } +func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { + podName := "k0s-token-generator-" + suffix := strings.Replace(nodeRole, "+", "-", -1) + podName += suffix // cleanup the pod if it already exists err := client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) @@ -223,8 +221,8 @@ func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, prim return string(podLogs), nil } -// generate the add node command from the join token, whether this is a primary node, and info from the embedded-cluster-config configmap -func generateAddNodeCommand(ctx context.Context, client kubernetes.Interface, primary bool, token string) ([]string, error) { +// generate the add node command from the join token, the node roles, and info from the embedded-cluster-config configmap +func generateAddNodeCommand(ctx context.Context, client kubernetes.Interface, nodeRole string, token string) ([]string, error) { cm, err := ReadConfigMap(client) if err != nil { return nil, fmt.Errorf("failed to read configmap: %w", err) @@ -239,13 +237,6 @@ func generateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr return nil, fmt.Errorf("failed to unmarshal cluster id %s: %w", clusterID, err) } - nodeRole := "" - if primary { - nodeRole = "controller" - } else { - nodeRole = "worker" - } - fullToken := joinToken{ ClusterID: clusterUUID, Token: token, @@ -257,5 +248,5 @@ func generateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr return nil, fmt.Errorf("unable to encode token: %w", err) } - return []string{binaryName, b64token}, nil + return []string{binaryName + " node join", b64token}, nil } From cee6046b569a3009e4b25b586a6314b97becbdeb Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Oct 2023 10:25:14 -0600 Subject: [PATCH 5/6] allow arbitrary node roles --- pkg/handlers/helmvm_node_join_command.go | 4 +- pkg/helmvm/node_join.go | 58 ++++++++++-------------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/pkg/handlers/helmvm_node_join_command.go b/pkg/handlers/helmvm_node_join_command.go index f300c8bda5..410d598fb2 100644 --- a/pkg/handlers/helmvm_node_join_command.go +++ b/pkg/handlers/helmvm_node_join_command.go @@ -22,7 +22,7 @@ func (h *Handler) GenerateHelmVMNodeJoinCommandSecondary(w http.ResponseWriter, return } - command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, false) + command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, "worker") if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) @@ -42,7 +42,7 @@ func (h *Handler) GenerateHelmVMNodeJoinCommandPrimary(w http.ResponseWriter, r return } - command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, true) + command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, "controller+worker") if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/helmvm/node_join.go b/pkg/helmvm/node_join.go index d2ed53dec4..4bbf1e197c 100644 --- a/pkg/helmvm/node_join.go +++ b/pkg/helmvm/node_join.go @@ -14,37 +14,34 @@ import ( "k8s.io/client-go/kubernetes" ) -var addPrimaryNodeMut = sync.Mutex{} -var addSecondaryNodeMut = sync.Mutex{} -var primaryNodeJoinCommand []string -var primaryNodeJoinCommandCreation *time.Time -var secondaryNodeJoinCommand []string -var secondaryNodeJoinCommandCreation *time.Time +type joinCommandEntry struct { + Command []string + Creation *time.Time + Mut sync.Mutex +} + +var joinCommandMapMut = sync.Mutex{} +var joinCommandMap = map[string]*joinCommandEntry{} // GenerateAddNodeCommand will generate the HelmVM node add command for a primary or secondary node // join commands will last for 24 hours, and will be cached for 1 hour after first generation -func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, primary bool) ([]string, *time.Time, error) { - if primary { - addPrimaryNodeMut.Lock() - defer addPrimaryNodeMut.Unlock() - } else { - addSecondaryNodeMut.Lock() - defer addSecondaryNodeMut.Unlock() +func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, nodeRole string) ([]string, *time.Time, error) { + // get the joinCommand struct entry for this node role + joinCommandMapMut.Lock() + if _, ok := joinCommandMap[nodeRole]; !ok { + joinCommandMap[nodeRole] = &joinCommandEntry{} } + joinCommand := joinCommandMap[nodeRole] + joinCommandMapMut.Unlock() - nodeRole := "" - if primary { - if primaryNodeJoinCommandCreation != nil && time.Now().Before(primaryNodeJoinCommandCreation.Add(time.Hour)) { - expiry := primaryNodeJoinCommandCreation.Add(time.Hour * 24) - return primaryNodeJoinCommand, &expiry, nil - } - nodeRole = "controller+worker" - } else { - if secondaryNodeJoinCommandCreation != nil && time.Now().Before(secondaryNodeJoinCommandCreation.Add(time.Hour)) { - expiry := secondaryNodeJoinCommandCreation.Add(time.Hour * 24) - return secondaryNodeJoinCommand, &expiry, nil - } - nodeRole = "worker" + // lock the joinCommand struct entry + joinCommand.Mut.Lock() + defer joinCommand.Mut.Unlock() + + // if the joinCommand has been generated in the past hour, return it + if joinCommand.Creation != nil && time.Now().Before(joinCommand.Creation.Add(time.Hour)) { + expiry := joinCommand.Creation.Add(time.Hour * 24) + return joinCommand.Command, &expiry, nil } newToken, err := runAddNodeCommandPod(ctx, client, nodeRole) @@ -58,13 +55,8 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, pr } now := time.Now() - if primary { - primaryNodeJoinCommand = newCmd - primaryNodeJoinCommandCreation = &now - } else { - secondaryNodeJoinCommand = newCmd - secondaryNodeJoinCommandCreation = &now - } + joinCommand.Command = newCmd + joinCommand.Creation = &now expiry := now.Add(time.Hour * 24) return newCmd, &expiry, nil From c73a5fcc2ea1c7dd5151de536a48aaabcecd2a7c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Oct 2023 10:50:21 -0600 Subject: [PATCH 6/6] role is controller not controller+worker --- pkg/handlers/helmvm_node_join_command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/handlers/helmvm_node_join_command.go b/pkg/handlers/helmvm_node_join_command.go index 410d598fb2..a17c39c0ef 100644 --- a/pkg/handlers/helmvm_node_join_command.go +++ b/pkg/handlers/helmvm_node_join_command.go @@ -42,7 +42,7 @@ func (h *Handler) GenerateHelmVMNodeJoinCommandPrimary(w http.ResponseWriter, r return } - command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, "controller+worker") + command, expiry, err := helmvm.GenerateAddNodeCommand(r.Context(), client, "controller") if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError)