diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 0a5fd52b2cdb..23a819e19a4c 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "log" "strconv" "time" @@ -262,10 +263,13 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + //TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) + //how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return r.handleClusterNotInitialized(ctx, scope) - } + log.Info("TESTING... skip handx``leClusterNotInitialized and push cluster for join") + //if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + // return r.handleClusterNotInitialized(ctx, scope) + //} // Every other case it's a join scenario // Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore them @@ -281,10 +285,12 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques // it's a control plane join if configOwner.IsControlPlaneMachine() { + log.Info("TESTING.... joinControlplane") return r.joinControlplane(ctx, scope) } // It's a worker join + log.Info("TESTING.... It's a worker join") return r.joinWorker(ctx, scope) } @@ -342,6 +348,9 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex // initialize the DataSecretAvailableCondition if missing. // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing // using the DataSecretGeneratedFailedReason + + scope.Info("TESTING.... In handleClusterNotInitialized") + if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") } @@ -424,6 +433,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex return ctrl.Result{}, err } + scope.Info("TESTING.... LookupOrGenerate new certificates") certificates := secret.NewCertificatesForInitialControlPlane(scope.Config.Spec.ClusterConfiguration) err = certificates.LookupOrGenerate( ctx, @@ -495,6 +505,10 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex } func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { + + scope.Info("TESTING.... joinWorker") + log.Println("TESTING.... joinWorker") + certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath) err := certificates.Lookup( ctx, @@ -600,6 +614,8 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S scope.Config.Spec.JoinConfiguration.ControlPlane = &bootstrapv1.JoinControlPlane{} } + scope.Info("TESTING.... NewControlPlaneJoinCerts") + log.Println("TESTING.... joinControlplane") certificates := secret.NewControlPlaneJoinCerts(scope.Config.Spec.ClusterConfiguration) err := certificates.Lookup( ctx, diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 081c57de3026..3bae9ba39e8f 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -261,10 +261,15 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * if config.ClusterConfiguration == nil { config.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} } + + //TODO: PCP-22 lookup or generate ca, sa, etcd certificates and key certificates := secret.NewCertificatesForInitialControlPlane(config.ClusterConfiguration) + //for _, certificate := range certificates { + // log.Info("TESTING.... lookup or generate ca, sa, etcd certificates and key: ", certificate) + //} controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane")) if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { - log.Error(err, "unable to lookup or create cluster certificates") + log.Error(err, "TESTING.... unable to lookup or create cluster certificates") conditions.MarkFalse(kcp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) return ctrl.Result{}, err } @@ -276,6 +281,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * return ctrl.Result{}, nil } + //TODO: PCP-22 adopt kubeconfig instead of generating new // Generate Cluster Kubeconfig if needed if result, err := r.reconcileKubeconfig(ctx, cluster, kcp); !result.IsZero() || err != nil { if err != nil { @@ -352,6 +358,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster * desiredReplicas := int(*kcp.Spec.Replicas) switch { + //TODO: PCP-22 skip creating new control plane // We are creating the first replica case numMachines < desiredReplicas && numMachines == 0: // Create new Machine w/ init @@ -528,6 +535,11 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context log := ctrl.LoggerFrom(ctx, "cluster", controlPlane.Cluster.Name) // If etcd is not managed by KCP this is a no-op. + if true { + //TODO: PCP-22 + return ctrl.Result{}, nil + } + if !controlPlane.IsEtcdManaged() { return ctrl.Result{}, nil } diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index 31034141ca9f..d9c6ee767baa 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -75,6 +75,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, // check if the kubeconfig secret was created by v1alpha2 controllers, and thus it has the Cluster as the owner instead of KCP; // if yes, adopt it. + //TODO: PCP-22 Need to inject original Kubeconfig instead of generating new if util.IsOwnedByObject(configSecret, cluster) && !util.IsControlledBy(configSecret, kcp) { if err := r.adoptKubeconfigSecret(ctx, cluster, configSecret, controllerOwnerRef); err != nil { return ctrl.Result{}, err diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index 1617827ff3e5..ee27bab1f3f5 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -106,11 +106,13 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c kcp.Status.ReadyReplicas = status.ReadyNodes kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes + //TODO: PCP-22 Initialized should be true to join new node to cluster or else it will try to init // This only gets initialized once and does not change if the kubeadm config map goes away. - if status.HasKubeadmConfig { - kcp.Status.Initialized = true - conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) - } + //if status.HasKubeadmConfig { + log.Info("TESTING.... set kcp.Status.Initialized to true") + kcp.Status.Initialized = true + conditions.MarkTrue(kcp, controlplanev1.AvailableCondition) + //} if kcp.Status.ReadyReplicas > 0 { kcp.Status.Ready = true diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 589c8a0a9db6..88e43b6a9f65 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -245,6 +245,9 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster if err != nil { return ctrl.Result{}, err } + + //TODO: PCP-22 + conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) if initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) } else { @@ -265,23 +268,44 @@ func (r *Reconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1 // Do not generate the Kubeconfig if there is a ControlPlaneRef, since the Control Plane provider is // responsible for the management of the Kubeconfig. We continue to manage it here only for backward // compatibility when a Control Plane provider is not in use. + + log.Info("TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef", "cluster.Spec.ControlPlaneRef", cluster.Spec.ControlPlaneRef) + //TODO: PCP-22 comment this to let secret generation for now, ControlPlaneRef is present already if cluster.Spec.ControlPlaneRef != nil { return ctrl.Result{}, nil } _, err := secret.Get(ctx, r.Client, util.ObjectKey(cluster), secret.Kubeconfig) - switch { - case apierrors.IsNotFound(err): - if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { - if err == kubeconfig.ErrDependentCertificateNotFound { - log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - return ctrl.Result{}, err + if err != nil { + log.Info("TESTING.... error getting kubeconfig", "err", err) + } + + // TODO: PCP-22 read kubeconfig secrets from kube-system namespace + log.Error(nil, "TESTING..... Do not generate the Kubeconfig if there is a ControlPlaneRef") + if err := kubeconfig.ReadSecret(ctx, r.Client, cluster); err != nil { + if err == kubeconfig.ErrDependentCertificateNotFound { + log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + if err == kubeconfig.ErrAlreadyExists { + log.Info("TESTING.... could not find secret for cluster, requesting", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - case err != nil: - return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) + return ctrl.Result{}, err } + //switch { + //case apierrors.IsNotFound(err): + // if err := kubeconfig.CreateSecret(ctx, r.Client, cluster); err != nil { + // if err == kubeconfig.ErrDependentCertificateNotFound { + // log.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) + // return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + // } + // return ctrl.Result{}, err + // } + //case err != nil: + // return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", cluster.Name, cluster.Namespace) + //} + return ctrl.Result{}, nil } diff --git a/util/kubeconfig/kubeconfig.go b/util/kubeconfig/kubeconfig.go index 2c8872bbb723..6e5332c0171e 100644 --- a/util/kubeconfig/kubeconfig.go +++ b/util/kubeconfig/kubeconfig.go @@ -22,6 +22,7 @@ import ( "crypto" "crypto/x509" "fmt" + "log" "time" "github.com/pkg/errors" @@ -41,6 +42,7 @@ import ( var ( // ErrDependentCertificateNotFound signals that a CA secret could not be found. ErrDependentCertificateNotFound = errors.New("could not find secret ca") + ErrAlreadyExists = errors.New("secrets \"t-cluster-kubeconfig\" already exists") ) // FromSecret fetches the Kubeconfig for a Cluster. @@ -107,6 +109,72 @@ func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Clust }) } +// ReadSecret reads the Kubeconfig secret from kube-system +func ReadSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error { + log.Println("TESTING.... IN ReadSecret") + + name := util.ObjectKey(cluster) + return ReadSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + }) +} + +// ReadSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. +func ReadSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { + //server := fmt.Sprintf("https://%s", endpoint) + //out, err := ReadExistingSecret(ctx, c, clusterName, server) + //if err != nil { + // return err + //} + log.Println("TESTING.... IN ReadSecretWithOwner") + + //clusterName := util.ObjectKey(clusterName) + configSecret, err := secret.GetFromNamespacedName(ctx, c, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: clusterName.Name}, secret.Kubeconfig) + if err != nil { + log.Println("TESTING....", "error in getting kubeconfig: ", err) + return err + } + + data, err := toKubeconfigBytes(configSecret) + if err != nil { + log.Println("TESTING....", "error in parsing kubeconfig: ", err) + return err + } + //if err := ReadExistingSecret(ctx, r.Client, configSecret); err != nil { + // return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig") + //} + + return c.Create(ctx, GenerateSecretWithOwner(clusterName, data, owner)) +} + +// ReadExistingSecret creates and stores a new Kubeconfig in the given secret. +func ReadExistingSecret(ctx context.Context, c client.Client, configSecret *corev1.Secret) error { + clusterName, _, err := secret.ParseSecretName(configSecret.Name) + if err != nil { + return errors.Wrap(err, "failed to parse secret name") + } + data, err := toKubeconfigBytes(configSecret) + if err != nil { + return err + } + + config, err := clientcmd.Load(data) + if err != nil { + return errors.Wrap(err, "failed to convert kubeconfig Secret into a clientcmdapi.Config") + } + endpoint := config.Clusters[clusterName].Server + key := client.ObjectKey{Name: clusterName, Namespace: configSecret.Namespace} + out, err := generateKubeconfig(ctx, c, key, endpoint) + if err != nil { + return err + } + configSecret.Data[secret.KubeconfigDataName] = out + return c.Update(ctx, configSecret) +} + // CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { server := fmt.Sprintf("https://%s", endpoint) @@ -237,6 +305,7 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client } func toKubeconfigBytes(out *corev1.Secret) ([]byte, error) { + //data, ok := out.Data[secret.KubeconfigDataName2] data, ok := out.Data[secret.KubeconfigDataName] if !ok { return nil, errors.Errorf("missing key %q in secret data", secret.KubeconfigDataName) diff --git a/util/secret/certificates.go b/util/secret/certificates.go index e3aadfbc7a37..06169e293d7b 100644 --- a/util/secret/certificates.go +++ b/util/secret/certificates.go @@ -24,6 +24,8 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" + "fmt" + "log" "math/big" "path/filepath" "strings" @@ -207,12 +209,75 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus return errors.WithStack(err) } // If a user has a badly formatted secret it will prevent the cluster from working. + //kp, err := secretToKeyPair(s, certificate.Purpose) kp, err := secretToKeyPair(s) if err != nil { return err } certificate.KeyPair = kp } + + //err := c.LookupKubeadm(ctx, ctrlclient, clusterName) + //if err != nil { + // log.Println("TESTING.... error:", err) + // return err + //} + return nil +} + +// LookupKubeadm looks up each certificate from secrets and populates the certificate with the secret data. +func (c Certificates) LookupKubeadm(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey) error { + // Look up each certificate as a secret and populate the certificate/key + + fmt.Println("TESTING.... LookupKubeadm") + for _, certificate := range c { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: Name(clusterName.Name, certificate.Purpose), + Namespace: clusterName.Namespace, + } + + //key := client.ObjectKey{ + // Name: "kubeadm-certs", + // Namespace: "kube-system", + //} + + if err := ctrlclient.Get(ctx, key, s); err != nil { + if apierrors.IsNotFound(err) { + if certificate.External { + //log.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) + //fmt.Println("TESTING.... ERROR external certificate not found: ", certificate.Purpose) + return errors.WithMessage(err, "external certificate not found") + } + + //fmt.Println("TESTING..... err:", err) + continue + } + //log.Println("TESTING.... ERROR: ", certificate.Purpose) + //fmt.Println("TESTING.... ERROR: ", certificate.Purpose) + return errors.WithStack(err) + } + + // If a user has a badly formatted secret it will prevent the cluster from working. + //log.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) + //fmt.Println("TESTING.... secretToKeyPair for certificate.Purpose", certificate.Purpose) + + //for k, _ := range s.Data { + // log.Println("TESTING.... s.Data", k) + //} + //for k, _ := range s.StringData { + // log.Println("TESTING.... s.StringData", k) + //} + //kp, err := secretToKeyPair(s, certificate.Purpose) + + kp, err := secretToKeyPair(s) + if err != nil { + return err + } + + certificate.KeyPair = kp + //certificate.Generated = true + } return nil } @@ -220,6 +285,7 @@ func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clus func (c Certificates) EnsureAllExist() error { for _, certificate := range c { if certificate.KeyPair == nil { + log.Println("TESTING....", certificate) return ErrMissingCertificate } if len(certificate.KeyPair.Cert) == 0 { @@ -238,6 +304,9 @@ func (c Certificates) EnsureAllExist() error { func (c Certificates) Generate() error { for _, certificate := range c { if certificate.KeyPair == nil { + //TODO: Read existing certificate and create required secrets + log.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) + fmt.Println("TESTING.... Certificate not present generate new for KeyPair", certificate.KeyFile) err := certificate.Generate() if err != nil { return err @@ -268,10 +337,10 @@ func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Cl return err } - // Generate the certificates that don't exist - if err := c.Generate(); err != nil { - return err - } + //// Generate the certificates that don't exist + //if err := c.Generate(); err != nil { + // return err + //} // Save any certificates that have been generated return c.SaveGenerated(ctx, ctrlclient, clusterName, owner) @@ -415,6 +484,51 @@ func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) { }, nil } +//func secretToKeyPair(s *corev1.Secret, purpose Purpose) (*certs.KeyPair, error) { +// +// var c, key []byte +// var exists bool +// var dataName, keyName string +// if purpose == ServiceAccount { +// dataName = string(purpose) + ".pub" +// keyName = string(purpose) + ".key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else if purpose == EtcdCA { +// dataName = "etcd-ca.crt" +// keyName = "etcd-ca.key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else if purpose == FrontProxyCA { +// dataName = "front-proxy-ca.crt" +// keyName = "front-proxy-ca.key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// +// } else { +// dataName = string(purpose) + ".crt" +// keyName = string(purpose) + ".key" +// log.Println("TESTING.... secret name: ", dataName, keyName) +// } +// +// c, exists = s.Data[dataName] +// if !exists { +// return nil, errors.Errorf("missing data for key %s", dataName) +// } +// //fmt.Println("TESTING.... c", c) +// // In some cases (external etcd) it's ok if the etcd.key does not exist. +// // TODO: some other function should ensure that the certificates we need exist. +// key, exists = s.Data[keyName] +// if !exists { +// key = []byte("") +// } +// //fmt.Println("TESTING.... key", key) +// +// return &certs.KeyPair{ +// Cert: c, +// Key: key, +// }, nil +//} + func generateCACert() (*certs.KeyPair, error) { x509Cert, privKey, err := newCertificateAuthority() if err != nil { diff --git a/util/secret/consts.go b/util/secret/consts.go index ccfe02ba2ab3..7ecd498cdfe2 100644 --- a/util/secret/consts.go +++ b/util/secret/consts.go @@ -22,6 +22,7 @@ type Purpose string const ( // KubeconfigDataName is the key used to store a Kubeconfig in the secret's data field. KubeconfigDataName = "value" + //KubeconfigDataName2 = "admin.conf" // TLSKeyDataName is the key used to store a TLS private key in the secret's data field. TLSKeyDataName = "tls.key" diff --git a/util/secret/secret.go b/util/secret/secret.go index 5af9fd9f5057..4face40a8529 100644 --- a/util/secret/secret.go +++ b/util/secret/secret.go @@ -38,7 +38,9 @@ func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName cli secret := &corev1.Secret{} secretKey := client.ObjectKey{ Namespace: clusterName.Namespace, - Name: Name(clusterName.Name, purpose), + //TODO: PCP-22 + //Name: string(purpose), + Name: Name(clusterName.Name, purpose), } if err := c.Get(ctx, secretKey, secret); err != nil {