From 120a582a78fa9d4ad6a2d7df91c502f74eec177c Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Tue, 22 Oct 2024 16:30:16 +0530 Subject: [PATCH] Support ControlPlaneKubeletLocalMode feature gate for k8sVersion >= 1.31.0 --- .../kubeadm/internal/controllers/scale.go | 15 ++++++ .../kubeadm/internal/controllers/upgrade.go | 4 ++ .../kubeadm/internal/workload_cluster.go | 46 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/controlplane/kubeadm/internal/controllers/scale.go b/controlplane/kubeadm/internal/controllers/scale.go index e467dd64d5c6..ee6dfb5112ad 100644 --- a/controlplane/kubeadm/internal/controllers/scale.go +++ b/controlplane/kubeadm/internal/controllers/scale.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "sigs.k8s.io/cluster-api/util/version" "strings" "github.com/blang/semver" @@ -54,6 +55,13 @@ func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Conte } bootstrapSpec := controlPlane.InitialControlPlaneConfig() + // We intentionally only parse major/minor/patch so that the subsequent code + // also already applies to beta versions of new releases. + parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) + } + internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant) fd := controlPlane.NextFailureDomainForScaleUp() if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil { logger.Error(err, "Failed to create initial control plane Machine") @@ -75,6 +83,13 @@ func (r *KubeadmControlPlaneReconciler) scaleUpControlPlane(ctx context.Context, // Create the bootstrap configuration bootstrapSpec := controlPlane.JoinControlPlaneConfig() + // We intentionally only parse major/minor/patch so that the subsequent code + // also already applies to beta versions of new releases. + parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) + } + internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant) fd := controlPlane.NextFailureDomainForScaleUp() if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil { logger.Error(err, "Failed to create additional control plane Machine") diff --git a/controlplane/kubeadm/internal/controllers/upgrade.go b/controlplane/kubeadm/internal/controllers/upgrade.go index a3c0a6f8ee56..942533c44b58 100644 --- a/controlplane/kubeadm/internal/controllers/upgrade.go +++ b/controlplane/kubeadm/internal/controllers/upgrade.go @@ -88,6 +88,10 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( if err := workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(ctx, imageRepository, parsedVersion); err != nil { return ctrl.Result{}, errors.Wrap(err, "failed to update the image repository in the kubeadm config map") } + + if err := workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec, parsedVersionTolerant); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to update feature gates in the kubeadm config map") + } } if kcp.Spec.KubeadmConfigSpec.ClusterConfiguration != nil && kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local != nil { diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index 99f3fc1b3524..23e8b8166cf6 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -89,6 +89,13 @@ var ( // ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes // to remove an etcd member. ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") + + // minKubernetesVersionControlPlaneKubeletLocalMode is the min version from which + // we will enable the ControlPlaneKubeletLocalMode kubeadm feature gate. + // Note: We have to do this with Kubernetes 1.31. Because with that version we encountered + // a case where it's not okay anymore to ignore the Kubernetes version skew (kubelet 1.31 uses + // the spec.clusterIP field selector that is only implemented in kube-apiserver >= 1.31.0). + minKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.31.0") ) // WorkloadCluster defines all behaviors necessary to upgrade kubernetes on a workload cluster @@ -107,6 +114,7 @@ type WorkloadCluster interface { ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error + UpdateFeatureGatesInKubeadmConfigMap(ctx context.Context, kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) error UpdateEtcdVersionInKubeadmConfigMap(ctx context.Context, imageRepository, imageTag string, version semver.Version) error UpdateEtcdExtraArgsInKubeadmConfigMap(ctx context.Context, extraArgs map[string]string, version semver.Version) error UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error @@ -181,6 +189,44 @@ func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, }, version) } +// UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map. +func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(ctx context.Context, kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) error { + return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) { + // We use DeepCopy here to avoid modifying the KCP object in the apiserver. + kubeadmConfigSpec := kubeadmConfigSpec.DeepCopy() + DefaultFeatureGates(kubeadmConfigSpec, kubernetesVersion) + + // Even if featureGates is nil, reset it to ClusterConfiguration + // to override any previously set feature gates. + c.FeatureGates = kubeadmConfigSpec.ClusterConfiguration.FeatureGates + }, kubernetesVersion) +} + +const ( + // ControlPlaneKubeletLocalMode is a feature gate of kubeadm that ensures + // kubelets only communicate with the local apiserver. + ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode" +) + +// DefaultFeatureGates defaults the feature gates field. +func DefaultFeatureGates(kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) { + if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) { + return + } + + if kubeadmConfigSpec.ClusterConfiguration == nil { + kubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} + } + + if kubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil { + kubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{} + } + + if _, ok := kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode]; !ok { + kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode] = true + } +} + // UpdateKubernetesVersionInKubeadmConfigMap updates the kubernetes version in the kubeadm config map. func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error { return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {