[WIP] PCP-22: Boostrap and CAPI changes

bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"log"
 	"strconv"
 	"time"
 
@@ -262,10 +263,13 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, nil
 	}
 
+	//TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join )
+	//how to make this condition true for new cluster as kubeadm cluster is already initialized
 	// Note: can't use IsFalse here because we need to handle the absence of the condition as well as false.
-	if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
-		return r.handleClusterNotInitialized(ctx, scope)
-	}
+	log.Info("TESTING... skip handx``leClusterNotInitialized and push cluster for join")
+	//if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
+	//	return r.handleClusterNotInitialized(ctx, scope)
+	//}
 
 	// Every other case it's a join scenario
 	// Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore them
@@ -281,10 +285,12 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// it's a control plane join
 	if configOwner.IsControlPlaneMachine() {
+		log.Info("TESTING.... joinControlplane")
 		return r.joinControlplane(ctx, scope)
 	}
 
 	// It's a worker join
+	log.Info("TESTING.... It's a worker join")
 	return r.joinWorker(ctx, scope)
 }
 
@@ -342,6 +348,9 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 	// initialize the DataSecretAvailableCondition if missing.
 	// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
 	// using the DataSecretGeneratedFailedReason
+
+	scope.Info("TESTING.... In handleClusterNotInitialized")
+
 	if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason {
 		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "")
 	}
@@ -424,6 +433,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 		return ctrl.Result{}, err
 	}
 
+	scope.Info("TESTING.... LookupOrGenerate new certificates")
 	certificates := secret.NewCertificatesForInitialControlPlane(scope.Config.Spec.ClusterConfiguration)
 	err = certificates.LookupOrGenerate(
 		ctx,
@@ -495,6 +505,10 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 }
 
 func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) {
+
+	scope.Info("TESTING.... joinWorker")
+	log.Println("TESTING.... joinWorker")
+
 	certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath)
 	err := certificates.Lookup(
 		ctx,
@@ -600,6 +614,8 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
 		scope.Config.Spec.JoinConfiguration.ControlPlane = &bootstrapv1.JoinControlPlane{}
 	}
 
+	scope.Info("TESTING.... NewControlPlaneJoinCerts")
+	log.Println("TESTING.... joinControlplane")
 	certificates := secret.NewControlPlaneJoinCerts(scope.Config.Spec.ClusterConfiguration)
 	err := certificates.Lookup(
 		ctx,

controlplane/kubeadm/internal/controllers/controller.go

@@ -261,10 +261,15 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	if config.ClusterConfiguration == nil {
 		config.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
 	}
+
+	//TODO: PCP-22 lookup or generate ca, sa, etcd certificates and key
 	certificates := secret.NewCertificatesForInitialControlPlane(config.ClusterConfiguration)
+	//for _, certificate := range certificates {
+	//	log.Info("TESTING.... lookup or generate ca, sa, etcd certificates and key: ", certificate)
+	//}
 	controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane"))
 	if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil {
-		log.Error(err, "unable to lookup or create cluster certificates")
+		log.Error(err, "TESTING.... unable to lookup or create cluster certificates")
 		conditions.MarkFalse(kcp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		return ctrl.Result{}, err
 	}
@@ -276,6 +281,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 		return ctrl.Result{}, nil
 	}
 
+	//TODO: PCP-22 adopt kubeconfig instead of generating new
 	// Generate Cluster Kubeconfig if needed
 	if result, err := r.reconcileKubeconfig(ctx, cluster, kcp); !result.IsZero() || err != nil {
 		if err != nil {
@@ -352,6 +358,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	desiredReplicas := int(*kcp.Spec.Replicas)
 
 	switch {
+	//TODO: PCP-22 skip creating new control plane
 	// We are creating the first replica
 	case numMachines < desiredReplicas && numMachines == 0:
 		// Create new Machine w/ init
@@ -528,6 +535,11 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context
 	log := ctrl.LoggerFrom(ctx, "cluster", controlPlane.Cluster.Name)
 
 	// If etcd is not managed by KCP this is a no-op.
+	if true {
+		//TODO: PCP-22
+		return ctrl.Result{}, nil
+	}
+
 	if !controlPlane.IsEtcdManaged() {
 		return ctrl.Result{}, nil
 	}

controlplane/kubeadm/internal/controllers/helpers.go

@@ -75,6 +75,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context,
 
 	// check if the kubeconfig secret was created by v1alpha2 controllers, and thus it has the Cluster as the owner instead of KCP;
 	// if yes, adopt it.
+	//TODO: PCP-22 Need to inject original Kubeconfig instead of generating new
 	if util.IsOwnedByObject(configSecret, cluster) && !util.IsControlledBy(configSecret, kcp) {
 		if err := r.adoptKubeconfigSecret(ctx, cluster, configSecret, controllerOwnerRef); err != nil {
 			return ctrl.Result{}, err

controlplane/kubeadm/internal/controllers/status.go

@@ -106,11 +106,13 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c
 	kcp.Status.ReadyReplicas = status.ReadyNodes
 	kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes
 
+	//TODO: PCP-22 Initialized should be true to join new node to cluster or else it will try to init
 	// This only gets initialized once and does not change if the kubeadm config map goes away.
-	if status.HasKubeadmConfig {
-		kcp.Status.Initialized = true
-		conditions.MarkTrue(kcp, controlplanev1.AvailableCondition)
-	}
+	//if status.HasKubeadmConfig {
+	log.Info("TESTING.... set kcp.Status.Initialized to true")
+	kcp.Status.Initialized = true
+	conditions.MarkTrue(kcp, controlplanev1.AvailableCondition)
+	//}
 
 	if kcp.Status.ReadyReplicas > 0 {
 		kcp.Status.Ready = true

internal/controllers/cluster/cluster_controller_phases.go

@@ -245,6 +245,9 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster
 		if err != nil {
 			return ctrl.Result{}, err
 		}
+
+		//TODO: PCP-22
+		conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
 		if initialized {
 			conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
 		} else {
@@ -265,23 +268,44 @@ func (r *Reconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1
 	// Do not generate the Kubeconfig if there is a ControlPlaneRef, since the Control Plane provider is
 	// responsible for the management of the Kubeconfig. We continue to manage it here only for backward
 	// compatibility when a Control Plane provider is not in use.
+
+	log.Info("TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef", "cluster.Spec.ControlPlaneRef", cluster.Spec.ControlPlaneRef)
+	//TODO: PCP-22 comment this to let secret generation for now, ControlPlaneRef is present already
 	if cluster.Spec.ControlPlaneRef != nil {
 		return ctrl.Result{}, nil
 	}
 
 	_, err := secret.Get(ctx, r.Client, util.ObjectKey(cluster), secret.Kubeconfig)
-	switch {
-	case apierrors.IsNotFound(err):
-		if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil {
-			if err == kubeconfig.ErrDependentCertificateNotFound {
-				log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA)
-				return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
-			}
-			return ctrl.Result{}, err
+	if err != nil {
+		log.Info("TESTING.... error getting kubeconfig", "err", err)
+	}
+
+	// TODO: PCP-22 read kubeconfig secrets from kube-system namespace
+	log.Error(nil, "TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef")
+	if err := kubeconfig.ReadSecret(ctx, r.Client, cluster); err != nil {
+		if err == kubeconfig.ErrDependentCertificateNotFound {
+			log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA)
+			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
+		}
+		if err == kubeconfig.ErrAlreadyExists {
+			log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA)
+			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
 		}
-	case err != nil:
-		return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace)
+		return ctrl.Result{}, err
 	}
 
+	//switch {
+	//case apierrors.IsNotFound(err):
+	//	if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil {
+	//		if err == kubeconfig.ErrDependentCertificateNotFound {
+	//			log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA)
+	//			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
+	//		}
+	//		return ctrl.Result{}, err
+	//	}
+	//case err != nil:
+	//	return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace)
+	//}
+
 	return ctrl.Result{}, nil
 }

util/kubeconfig/kubeconfig.go

@@ -22,6 +22,7 @@ import (
 	"crypto"
 	"crypto/x509"
 	"fmt"
+	"log"
 	"time"
 
 	"github.com/pkg/errors"
@@ -41,6 +42,7 @@ import (
 var (
 	// ErrDependentCertificateNotFound signals that a CA secret could not be found.
 	ErrDependentCertificateNotFound = errors.New("could not find secret ca")
+	ErrAlreadyExists                = errors.New("secrets \"t-cluster-kubeconfig\" already exists")
 )
 
 // FromSecret fetches the Kubeconfig for a Cluster.
@@ -107,6 +109,72 @@ func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Clust
 	})
 }
 
+// ReadSecret reads the Kubeconfig secret from kube-system
+func ReadSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error {
+	log.Println("TESTING.... IN ReadSecret")
+
+	name := util.ObjectKey(cluster)
+	return ReadSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{
+		APIVersion: clusterv1.GroupVersion.String(),
+		Kind:       "Cluster",
+		Name:       cluster.Name,
+		UID:        cluster.UID,
+	})
+}
+
+// ReadSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference.
+func ReadSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error {
+	//server := fmt.Sprintf("https://%s", endpoint)
+	//out, err := ReadExistingSecret(ctx, c, clusterName, server)
+	//if err != nil {
+	//	return err
+	//}
+	log.Println("TESTING.... IN ReadSecretWithOwner")
+
+	//clusterName := util.ObjectKey(clusterName)
+	configSecret, err := secret.GetFromNamespacedName(ctx, c, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: clusterName.Name}, secret.Kubeconfig)
+	if err != nil {
+		log.Println("TESTING....", "error in getting kubeconfig: ", err)
+		return err
+	}
+
+	data, err := toKubeconfigBytes(configSecret)
+	if err != nil {
+		log.Println("TESTING....", "error in parsing kubeconfig: ", err)
+		return err
+	}
+	//if err := ReadExistingSecret(ctx, r.Client, configSecret); err != nil {
+	//	return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig")
+	//}
+
+	return c.Create(ctx, GenerateSecretWithOwner(clusterName, data, owner))
+}
+
+// ReadExistingSecret creates and stores a new Kubeconfig in the given secret.
+func ReadExistingSecret(ctx context.Context, c client.Client, configSecret *corev1.Secret) error {
+	clusterName, _, err := secret.ParseSecretName(configSecret.Name)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse secret name")
+	}
+	data, err := toKubeconfigBytes(configSecret)
+	if err != nil {
+		return err
+	}
+
+	config, err := clientcmd.Load(data)
+	if err != nil {
+		return errors.Wrap(err, "failed to convert kubeconfig Secret into a clientcmdapi.Config")
+	}
+	endpoint := config.Clusters[clusterName].Server
+	key := client.ObjectKey{Name: clusterName, Namespace: configSecret.Namespace}
+	out, err := generateKubeconfig(ctx, c, key, endpoint)
+	if err != nil {
+		return err
+	}
+	configSecret.Data[secret.KubeconfigDataName] = out
+	return c.Update(ctx, configSecret)
+}
+
 // CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference.
 func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error {
 	server := fmt.Sprintf("https://%s", endpoint)
@@ -237,6 +305,7 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client
 }
 
 func toKubeconfigBytes(out *corev1.Secret) ([]byte, error) {
+	//data, ok := out.Data[secret.KubeconfigDataName2]
 	data, ok := out.Data[secret.KubeconfigDataName]
 	if !ok {
 		return nil, errors.Errorf("missing key %q in secret data", secret.KubeconfigDataName)