Skip to content

Commit

Permalink
Merge pull request #209 from spectrocloud/PCP-3412
Browse files Browse the repository at this point in the history
PCP-3412: Fix cloud-init for 1.31 cluster
  • Loading branch information
Kun483 authored Oct 17, 2024
2 parents db593ce + c03d1a3 commit b084908
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 11 deletions.
23 changes: 20 additions & 3 deletions bootstrap/kubeadm/internal/cloudinit/cloudinit.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ import (
"fmt"
"text/template"

"github.com/blang/semver"
"github.com/pkg/errors"

bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
"sigs.k8s.io/cluster-api/util/version"
)

const (
Expand Down Expand Up @@ -56,6 +58,7 @@ type BaseUserData struct {
KubeadmCommand string
KubeadmVerbosity string
SentinelFileCommand string
KubernetesVersion semver.Version
}

func (input *BaseUserData) prepare() error {
Expand All @@ -64,7 +67,7 @@ func (input *BaseUserData) prepare() error {
input.KubeadmCommand = fmt.Sprintf(standardJoinCommand, input.KubeadmVerbosity)
if input.UseExperimentalRetry {
input.KubeadmCommand = retriableJoinScriptName
joinScriptFile, err := generateBootstrapScript(input)
joinScriptFile, err := generateBootstrapScript(input, input.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "failed to generate user data for machine joining control plane")
}
Expand Down Expand Up @@ -118,12 +121,22 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) {
}

var (
//go:embed kubeadm-bootstrap-script-pre-k8s-1-31.sh
kubeadmBootstrapScriptPre1_31 string
//go:embed kubeadm-bootstrap-script.sh
kubeadmBootstrapScript string

// kubernetesVersion1_31 is the version where kubeadm removed the update-status phase
// and introduced new phases with the ControlPlaneKubeletLocalMode feature gate.
kubernetesVersion1_31 = semver.MustParse("1.31.0")
)

func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) {
joinScript, err := generate("JoinScript", kubeadmBootstrapScript, input)
func generateBootstrapScript(input interface{}, parsedversion semver.Version) (*bootstrapv1.File, error) {
bootstrapScript := kubeadmBootstrapScript
if useKubeadmBootstrapScriptPre1_31(parsedversion) {
bootstrapScript = kubeadmBootstrapScriptPre1_31
}
joinScript, err := generate("JoinScript", bootstrapScript, input)
if err != nil {
return nil, errors.Wrap(err, "failed to bootstrap script for machine joins")
}
Expand All @@ -134,3 +147,7 @@ func generateBootstrapScript(input interface{}) (*bootstrapv1.File, error) {
Content: string(joinScript),
}, nil
}

func useKubeadmBootstrapScriptPre1_31(parsedversion semver.Version) bool {
return version.Compare(parsedversion, kubernetesVersion1_31, version.WithoutPreReleases()) < 0
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#!/bin/bash
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Log an error and exit.
# Args:
# $1 Message to log with the error
# $2 The error code to return
log::error_exit() {
local message="${1}"
local code="${2}"

log::error "${message}"
# {{ if .ControlPlane }}
log::info "Removing member from cluster status"
kubeadm reset -f update-cluster-status || true
log::info "Removing etcd member"
kubeadm reset -f remove-etcd-member || true
# {{ end }}
log::info "Resetting kubeadm"
kubeadm reset -f || true
log::error "cluster.x-k8s.io kubeadm bootstrap script $0 exiting with status ${code}"
exit "${code}"
}

log::success_exit() {
log::info "cluster.x-k8s.io kubeadm bootstrap script $0 finished"
exit 0
}

# Log an error but keep going.
log::error() {
local message="${1}"
timestamp=$(date --iso-8601=seconds)
echo "!!! [${timestamp}] ${1}" >&2
shift
for message; do
echo " ${message}" >&2
done
}

# Print a status line. Formatted to show up in a stream of output.
log::info() {
timestamp=$(date --iso-8601=seconds)
echo "+++ [${timestamp}] ${1}"
shift
for message; do
echo " ${message}"
done
}

check_kubeadm_command() {
local command="${1}"
local code="${2}"
case ${code} in
"0")
log::info "kubeadm reported successful execution for ${command}"
;;
"1")
log::error "kubeadm reported failed action(s) for ${command}"
;;
"2")
log::error "kubeadm reported preflight check error during ${command}"
;;
"3")
log::error_exit "kubeadm reported validation error for ${command}" "${code}"
;;
*)
log::error "kubeadm reported unknown error ${code} for ${command}"
;;
esac
}

function retry-command() {
n=0
local kubeadm_return
until [ $n -ge 5 ]; do
log::info "running '$*'"
# shellcheck disable=SC1083
"$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}}
kubeadm_return=$?
check_kubeadm_command "'$*'" "${kubeadm_return}"
if [ ${kubeadm_return} -eq 0 ]; then
break
fi
# We allow preflight errors to pass
if [ ${kubeadm_return} -eq 2 ]; then
break
fi
n=$((n + 1))
sleep 15
done
if [ ${kubeadm_return} -ne 0 ]; then
log::error_exit "too many errors, exiting" "${kubeadm_return}"
fi
}

# {{ if .ControlPlane }}
function try-or-die-command() {
local kubeadm_return
log::info "running '$*'"
# shellcheck disable=SC1083
"$@" --config=/run/kubeadm/kubeadm-join-config.yaml {{.KubeadmVerbosity}}
kubeadm_return=$?
check_kubeadm_command "'$*'" "${kubeadm_return}"
if [ ${kubeadm_return} -ne 0 ]; then
log::error_exit "fatal error, exiting" "${kubeadm_return}"
fi
}
# {{ end }}

retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests
# {{ if .ControlPlane }}
retry-command kubeadm join phase control-plane-prepare download-certs
retry-command kubeadm join phase control-plane-prepare certs
retry-command kubeadm join phase control-plane-prepare kubeconfig
retry-command kubeadm join phase control-plane-prepare control-plane
# {{ end }}
retry-command kubeadm join phase kubelet-start
# {{ if .ControlPlane }}
try-or-die-command kubeadm join phase control-plane-join etcd
retry-command kubeadm join phase control-plane-join update-status
retry-command kubeadm join phase control-plane-join mark-control-plane
# {{ end }}

log::success_exit
12 changes: 4 additions & 8 deletions bootstrap/kubeadm/internal/cloudinit/kubeadm-bootstrap-script.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2020 The Kubernetes Authors.
# Copyright 2024 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -106,7 +106,6 @@ function retry-command() {
fi
}

# {{ if .ControlPlane }}
function try-or-die-command() {
local kubeadm_return
log::info "running '$*'"
Expand All @@ -118,7 +117,6 @@ function try-or-die-command() {
log::error_exit "fatal error, exiting" "${kubeadm_return}"
fi
}
# {{ end }}

retry-command kubeadm join phase preflight --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests
# {{ if .ControlPlane }}
Expand All @@ -128,10 +126,8 @@ retry-command kubeadm join phase control-plane-prepare kubeconfig
retry-command kubeadm join phase control-plane-prepare control-plane
# {{ end }}
retry-command kubeadm join phase kubelet-start
# {{ if .ControlPlane }}
try-or-die-command kubeadm join phase control-plane-join etcd
retry-command kubeadm join phase control-plane-join update-status
retry-command kubeadm join phase control-plane-join mark-control-plane
# {{ end }}

# Run kubeadm join and skip all already executed phases.
try-or-die-command kubeadm join --skip-phases preflight,control-plane-prepare,kubelet-start

log::success_exit
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
Mounts: scope.Config.Spec.Mounts,
DiskSetup: scope.Config.Spec.DiskSetup,
KubeadmVerbosity: verbosityFlag,
KubernetesVersion: parsedVersion,
},
InitConfiguration: initdata,
ClusterConfiguration: clusterdata,
Expand Down Expand Up @@ -589,6 +590,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
DiskSetup: scope.Config.Spec.DiskSetup,
KubeadmVerbosity: verbosityFlag,
UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
KubernetesVersion: parsedVersion,
},
JoinConfiguration: joinData,
}
Expand Down Expand Up @@ -693,6 +695,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
DiskSetup: scope.Config.Spec.DiskSetup,
KubeadmVerbosity: verbosityFlag,
UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
KubernetesVersion: parsedVersion,
},
}

Expand Down

0 comments on commit b084908

Please sign in to comment.