From 6e856808ce44e41aaf0487272e6fa76082ab77c8 Mon Sep 17 00:00:00 2001 From: emosbaugh Date: Wed, 20 Sep 2023 04:10:56 +0000 Subject: [PATCH] Create new Containerd version --- addons/containerd/1.6.24/Manifest | 10 + addons/containerd/1.6.24/crictl.yaml | 2 + addons/containerd/1.6.24/host-preflight.yaml | 52 +++ addons/containerd/1.6.24/install.sh | 345 ++++++++++++++++++ .../1.6.24/kubeadm-init-config-v1beta2.yaml | 9 + .../1.6.24/kubeadm-join-config-v1beta2.yaml | 9 + hack/testdata/manifest/clean | 2 +- scripts/Manifest | 2 +- web/src/installers/versions.js | 2 +- 9 files changed, 430 insertions(+), 3 deletions(-) create mode 100644 addons/containerd/1.6.24/Manifest create mode 100644 addons/containerd/1.6.24/crictl.yaml create mode 100644 addons/containerd/1.6.24/host-preflight.yaml create mode 100644 addons/containerd/1.6.24/install.sh create mode 100644 addons/containerd/1.6.24/kubeadm-init-config-v1beta2.yaml create mode 100644 addons/containerd/1.6.24/kubeadm-join-config-v1beta2.yaml diff --git a/addons/containerd/1.6.24/Manifest b/addons/containerd/1.6.24/Manifest new file mode 100644 index 0000000000..116211bd73 --- /dev/null +++ b/addons/containerd/1.6.24/Manifest @@ -0,0 +1,10 @@ +yum libzstd +asset runc https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64 +dockerout rhel-7 addons/containerd/template/Dockerfile.centos7 1.6.24 +dockerout rhel-7-force addons/containerd/template/Dockerfile.centos7-force 1.6.24 +dockerout rhel-8 addons/containerd/template/Dockerfile.centos8 1.6.24 +dockerout rhel-9 addons/containerd/template/Dockerfile.rhel9 1.6.24 +dockerout ubuntu-18.04 addons/containerd/template/Dockerfile.ubuntu18 1.6.21 +dockerout ubuntu-20.04 addons/containerd/template/Dockerfile.ubuntu20 1.6.24 +dockerout ubuntu-22.04 addons/containerd/template/Dockerfile.ubuntu22 1.6.24 +image pause registry.k8s.io/pause:3.6 diff --git a/addons/containerd/1.6.24/crictl.yaml b/addons/containerd/1.6.24/crictl.yaml new file mode 100644 index 0000000000..428f16539b --- /dev/null +++ b/addons/containerd/1.6.24/crictl.yaml @@ -0,0 +1,2 @@ +runtime-endpoint: unix:///var/run/containerd/containerd.sock +image-endpoint: unix:///var/run/containerd/containerd.sock diff --git a/addons/containerd/1.6.24/host-preflight.yaml b/addons/containerd/1.6.24/host-preflight.yaml new file mode 100644 index 0000000000..adb0c82f0e --- /dev/null +++ b/addons/containerd/1.6.24/host-preflight.yaml @@ -0,0 +1,52 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: HostPreflight +metadata: + name: kurl-builtin +spec: + collectors: + - hostOS: {} + analyzers: + - hostOS: + outcomes: + - pass: + when: "centos = 7" + message: "containerd addon supports centos 7" + - pass: + when: "rhel = 7" + message: "containerd addon supports rhel 7" + - pass: + when: "ol = 7" + message: "containerd addon supports ol 7" + - pass: + when: "centos = 8" + message: "containerd addon supports centos 8" + - pass: + when: "rhel = 8" + message: "containerd addon supports rhel 8" + - pass: + when: "ol = 8" + message: "containerd addon supports ol 8" + - pass: + when: "centos = 9" + message: "containerd addon supports centos 9" + - pass: + when: "rhel = 9" + message: "containerd addon supports rhel 9" + - pass: + when: "rocky = 9" + message: "containerd addon supports rocky 9" + - fail: + when: "ol = 9" + message: "containerd addon does not support ol 9" + - fail: + when: "ubuntu = 16.04" + message: "containerd addon does not support ubuntu 16.04" + - warn: + when: "ubuntu = 18.04" + message: "containerd addon supports ubuntu 18.04, but only up to containerd 1.6.21, which will be installed instead of 1.6.24" + - pass: + when: "ubuntu = 20.04" + message: "containerd addon supports ubuntu 20.04" + - pass: + when: "ubuntu = 22.04" + message: "containerd addon supports ubuntu 22.04" diff --git a/addons/containerd/1.6.24/install.sh b/addons/containerd/1.6.24/install.sh new file mode 100644 index 0000000000..638aa6098f --- /dev/null +++ b/addons/containerd/1.6.24/install.sh @@ -0,0 +1,345 @@ +# shellcheck disable=SC2148 + +CONTAINERD_NEEDS_RESTART=0 +CONTAINERD_DID_MIGRATE_FROM_DOCKER=0 + +function containerd_pre_init() { + local src="$DIR/addons/containerd/$CONTAINERD_VERSION" + + # Explicitly configure kubelet to use containerd instead of detecting dockershim socket + if [ -d "$DIR/kustomize/kubeadm/init-patches" ]; then + cp "$src/kubeadm-init-config-v1beta2.yaml" "$DIR/kustomize/kubeadm/init-patches/containerd-kubeadm-init-config-v1beta2.yml" + fi + + containerd_host_init +} + +function containerd_join() { + local src="$DIR/addons/containerd/$CONTAINERD_VERSION" + + # Explicitly configure kubelet to use containerd instead of detecting dockershim socket + if [ -d "$DIR/kustomize/kubeadm/join-patches" ]; then + cp "$src/kubeadm-join-config-v1beta2.yaml" "$DIR/kustomize/kubeadm/join-patches/containerd-kubeadm-join-config-v1beta2.yml" + fi + + containerd_host_init +} + +function containerd_install() { + local src="$DIR/addons/containerd/$CONTAINERD_VERSION" + + if ! containerd_xfs_ftype_enabled; then + bail "The filesystem mounted at /var/lib/containerd does not have ftype enabled" + fi + + containerd_migrate_from_docker + + install_host_packages "$src" containerd.io + + chmod +x ${DIR}/addons/containerd/${CONTAINERD_VERSION}/assets/runc + # If the runc binary is executing the cp command will fail with "text file busy" error. + # Containerd uses runc in detached mode so any runc processes should be short-lived and exit + # as soon as the container starts + try_1m_stderr cp ${DIR}/addons/containerd/${CONTAINERD_VERSION}/assets/runc $(which runc) + + logStep "Containerd configuration" + containerd_configure + + log "Enabling containerd" + systemctl enable containerd + + log "Enabling containerd crictl" + containerd_configure_ctl "$src" + + log "Checking for containerd custom settings" + containerd_configure_limitnofile + + # NOTE: this will not remove the proxy + log "Checking for proxy set" + if [ -n "$PROXY_ADDRESS" ]; then + log "Proxy is set with the value: ($PROXY_ADDRESS)" + containerd_configure_proxy + fi + + log "Checking registry configuration for the distro ${K8S_DISTRO} and if Docker registry IP is set" + if commandExists ${K8S_DISTRO}_registry_containerd_configure && [ -n "$DOCKER_REGISTRY_IP" ]; then + log "Docker registry IP is set with the value: ($DOCKER_REGISTRY_IP)" + ${K8S_DISTRO}_registry_containerd_configure "$DOCKER_REGISTRY_IP" + CONTAINERD_NEEDS_RESTART=1 + fi + + log "Checking if containerd requires to be re-started" + if [ "$CONTAINERD_NEEDS_RESTART" = "1" ]; then + log "Re-starting containerd" + systemctl daemon-reload + restart_systemd_and_wait containerd + CONTAINERD_NEEDS_RESTART=0 + fi + + logSuccess "Containerd is successfully configured" + + log "Checking if is required to migrate images from Docker" + if [ "$AIRGAP" = "1" ] && [ "$CONTAINERD_DID_MIGRATE_FROM_DOCKER" = "1" ]; then + logStep "Migrating images from Docker to Containerd..." + containerd_migrate_images_from_docker + logSuccess "Images migrated successfully" + fi + + load_images $src/images + + log "Checking if the kubelet service is enabled" + if systemctl list-unit-files | grep -v disabled | grep -q kubelet.service ; then + # do not try to start and wait for the kubelet if it is not yet configured + if [ -f /etc/kubernetes/kubelet.conf ]; then + log "Starting kubectl" + systemctl start kubelet + # If using the internal load balancer the Kubernetes API server will be unavailable until + # kubelet starts the HAProxy static pod. This check ensures the Kubernetes API server + # is available before proceeeding. + # "nodes.v1." is needed becasue addons can have a CRD names "nodes", like nodes.longhorn.io + try_5m kubectl --kubeconfig=/etc/kubernetes/kubelet.conf get nodes.v1. + fi + fi +} + +function containerd_host_init() { + containerd_install_libzstd_if_missing +} + +function containerd_install_libzstd_if_missing() { + local src="$DIR/addons/containerd/$CONTAINERD_VERSION" + + case "$LSB_DIST" in + centos|rhel|ol|rocky|amzn) + if yum_is_host_package_installed libzstd ; then + return + fi + + if is_rhel_9_variant ; then + yum_ensure_host_package libzstd + else + yum_install_host_archives "$src" libzstd + fi + ;; + esac +} + +function containerd_configure() { + if [ "$CONTAINERD_PRESERVE_CONFIG" = "1" ]; then + echo "Skipping containerd configuration in order to preserve config." + return + fi + mkdir -p /etc/containerd + containerd config default > /etc/containerd/config.toml + + sed -i '/systemd_cgroup/d' /etc/containerd/config.toml + sed -i '/containerd.runtimes.runc.options/d' /etc/containerd/config.toml + sed -i 's/level = ""/level = "warn"/' /etc/containerd/config.toml + cat >> /etc/containerd/config.toml < "$tmp" + "$DIR/bin/toml" -basefile=/etc/containerd/config.toml -patchfile="$tmp" + fi + + CONTAINERD_NEEDS_RESTART=1 +} + +function containerd_configure_ctl() { + local src="$1" + + log "Checks if the file /etc/crictl.yaml exist" + if [ -e "/etc/crictl.yaml" ]; then + log "Found /etc/crictl.yaml" + return 0 + fi + + log "Creates /etc/crictl.yaml" + cp "$src/crictl.yaml" /etc/crictl.yaml +} + +# ceph mon fails liveness on rhel 9 and variants without this setting +# https://github.com/rook/rook/issues/10110 +# https://github.com/coreos/fedora-coreos-tracker/issues/329 +function containerd_configure_limitnofile() { + local file=/etc/systemd/system/containerd.service.d/override-limitnofile.conf + + if [ -f "$file" ]; then + log "Found /etc/systemd/system/containerd.service.d/override-limitnofile.conf" + return + fi + + log "Creating /etc/systemd/system/containerd.service.d" + mkdir -p /etc/systemd/system/containerd.service.d + + echo "# Generated by kURL" > "$file" + echo "[Service]" >> "$file" + echo "LimitNOFILE=1048576" >> "$file" +} + +function containerd_configure_proxy() { + log "Configuring containerd proxy" + local previous_http_proxy="$(cat /etc/systemd/system/containerd.service.d/http-proxy.conf 2>/dev/null | grep -io 'http_proxy=[^\" ]*' | awk 'BEGIN { FS="=" }; { print $2 }')" + local previous_https_proxy="$(cat /etc/systemd/system/containerd.service.d/http-proxy.conf 2>/dev/null | grep -io 'https_proxy=[^\" ]*' | awk 'BEGIN { FS="=" }; { print $2 }')" + local previous_no_proxy="$(cat /etc/systemd/system/containerd.service.d/http-proxy.conf 2>/dev/null | grep -io 'no_proxy=[^\" ]*' | awk 'BEGIN { FS="=" }; { print $2 }')" + log "Previous http proxy: ($previous_http_proxy)" + log "Previous https proxy: ($previous_https_proxy)" + log "Previous no proxy: ($previous_no_proxy)" + if [ "$PROXY_ADDRESS" = "$previous_proxy" ] && [ "$PROXY_HTTPS_ADDRESS" = "$previous_https_proxy" ] && [ "$NO_PROXY_ADDRESSES" = "$previous_no_proxy" ]; then + log "No changes were found. Proxy configuration still the same" + return + fi + + log "Updating proxy configuration: HTTP_PROXY=${PROXY_ADDRESS} NO_PROXY=${NO_PROXY_ADDRESSES}" + mkdir -p /etc/systemd/system/containerd.service.d + local file=/etc/systemd/system/containerd.service.d/http-proxy.conf + + echo "# Generated by kURL" > $file + echo "[Service]" >> $file + + echo "Environment=\"HTTP_PROXY=${PROXY_ADDRESS}\" \"HTTPS_PROXY=${PROXY_HTTPS_ADDRESS}\" \"NO_PROXY=${NO_PROXY_ADDRESSES}\"" >> $file + + + CONTAINERD_NEEDS_RESTART=1 +} + +# Returns 0 on non-xfs filesystems and on xfs filesystems if ftype=1. +function containerd_xfs_ftype_enabled() { + if ! commandExists xfs_info; then + return 0 + fi + + mkdir -p /var/lib/containerd + + if xfs_info /var/lib/containerd 2>/dev/null | grep -q "ftype=0"; then + return 1 + fi + + return 0 +} + +function containerd_migrate_from_docker() { + if ! commandExists docker; then + return + fi + + if ! commandExists kubectl; then + return + fi + + local kubeconfigFlag="--kubeconfig=/etc/kubernetes/kubelet.conf" + + if ! kubectl "$kubeconfigFlag" get node "$(get_local_node_name)" -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' 2>/dev/null | grep -q docker ; then + return + fi + + # steps from https://kubernetes.io/docs/tasks/administer-cluster/migrating-from-dockershim/change-runtime-containerd/ + + echo "Draining node to prepare for migration from docker to containerd" + + # Delete pods that depend on other pods on the same node + if [ -f "$DIR/addons/ekco/$EKCO_VERSION/reboot/shutdown.sh" ]; then + bash $DIR/addons/ekco/$EKCO_VERSION/reboot/shutdown.sh + elif [ -f /opt/ekco/shutdown.sh ]; then + bash /opt/ekco/shutdown.sh + else + logFail "EKCO shutdown script not available. Migration to containerd may fail\n" + if ! confirmN ; then + bail "Migration to Containerd has been aborted." + fi + fi + + echo "Cordoning node" + + local node= + node="$(get_local_node_name)" + kubectl "$kubeconfigFlag" cordon "$node" + + echo "Deleting pods" + local allPodUIDs=$(kubectl "$kubeconfigFlag" get pods --all-namespaces -ojsonpath='{ range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.namespace}{"\n"}{end}') + + # Drain remaining pods using only the permissions available to kubelet + while read -r uid; do + local pod=$(echo "${allPodUIDs[*]}" | grep "$uid") + if [ -z "$pod" ]; then + continue + fi + local podName=$(echo "$pod" | awk '{ print $1 }') + local podNamespace=$(echo "$pod" | awk '{ print $3 }') + # some may timeout but proceed anyway + kubectl "$kubeconfigFlag" delete pod "$podName" --namespace="$podNamespace" --timeout=60s || true + done < <(ls /var/lib/kubelet/pods) + + echo "Stopping kubelet" + systemctl stop kubelet + + if kubectl "$kubeconfigFlag" get node "$node" -ojsonpath='{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io/cri-socket}' | grep -q "dockershim.sock" ; then + kubectl "$kubeconfigFlag" annotate node "$node" --overwrite "kubeadm.alpha.kubernetes.io/cri-socket=unix:///run/containerd/containerd.sock" + fi + + if [ "$(docker ps -aq | wc -l)" != "0" ] ; then + docker ps -aq | xargs docker rm -f || true + fi + + # Reconfigure kubelet to use containerd + containerdFlags="--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" + sed -i "s@\(KUBELET_KUBEADM_ARGS=\".*\)\"@\1 $containerdFlags\" @" /var/lib/kubelet/kubeadm-flags.env + + systemctl daemon-reload + + echo "Migrated to containerd" + CONTAINERD_DID_MIGRATE_FROM_DOCKER=1 +} + +function containerd_can_migrate_images_from_docker() { + local images_kb="$(du -sc /var/lib/docker/overlay2 | grep total | awk '{print $1}')" + local available_kb="$(df --output=avail /var/lib/containerd/ | awk 'NR > 1')" + + if [ -z "$images_kb" ]; then + logWarn "Unable to determine size of Docker images" + return 0 + elif [ -z "$available_kb" ]; then + logWarn "Unable to determine available disk space in /var/lib/containerd/" + return 0 + else + local images_kb_x2="$(expr $images_kb + $images_kb)" + if [ "$available_kb" -lt "$images_kb_x2" ]; then + local images_human="$(echo "$images_kb" | awk '{print int($1/1024/1024+0.5) "GB"}')" + local available_human="$(echo "$available_kb" | awk '{print int($1/1024/1024+0.5) "GB"}')" + logFail "There is not enough available disk space (${available_human}) to migrate images (${images_human}) from Docker to Containerd." + logFail "Please make sure there is at least 2 x size of Docker images available disk space." + return 1 + fi + fi + return 0 +} + +function containerd_migrate_images_from_docker() { + if ! containerd_can_migrate_images_from_docker ; then + exit 1 + fi + + # we must always clean up $tmpdir since it can take up a lot of space + local errcode=0 + local tmpdir="$(mktemp -d -p /var/lib/containerd)" + _containerd_migrate_images_from_docker "$tmpdir" || errcode="$?" + rm -rf "$tmpdir" + return "$errcode" +} + +function _containerd_migrate_images_from_docker() { + local tmpdir="$1" + local imagefile= + for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v '^'); do + imagefile="${tmpdir}/$(echo $image | tr -cd '[:alnum:]').tar" + (set -x; docker save $image -o "$imagefile") + done + for image in $tmpdir/* ; do + (set -x; ctr -n=k8s.io images import $image) + done +} diff --git a/addons/containerd/1.6.24/kubeadm-init-config-v1beta2.yaml b/addons/containerd/1.6.24/kubeadm-init-config-v1beta2.yaml new file mode 100644 index 0000000000..539743bc97 --- /dev/null +++ b/addons/containerd/1.6.24/kubeadm-init-config-v1beta2.yaml @@ -0,0 +1,9 @@ +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +metadata: + name: kubeadm-init-configuration +nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + container-runtime: remote + container-runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/addons/containerd/1.6.24/kubeadm-join-config-v1beta2.yaml b/addons/containerd/1.6.24/kubeadm-join-config-v1beta2.yaml new file mode 100644 index 0000000000..d3c3d7a676 --- /dev/null +++ b/addons/containerd/1.6.24/kubeadm-join-config-v1beta2.yaml @@ -0,0 +1,9 @@ +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +metadata: + name: kubeadm-join-configuration +nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + container-runtime: remote + container-runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/hack/testdata/manifest/clean b/hack/testdata/manifest/clean index a0189efb3b..0093566799 100644 --- a/hack/testdata/manifest/clean +++ b/hack/testdata/manifest/clean @@ -13,5 +13,5 @@ STEP_VERSIONS=(0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 # ROOK_STEP_VERSIONS array is generated by the server and injected at runtime based on supported rook versions ROOK_STEP_VERSIONS=(1.0.4-14.2.21 0.0.0 0.0.0 0.0.0 1.4.9 1.5.12 1.6.11 1.7.11 1.8.10 1.9.12 1.10.11 1.11.8 1.12.4) # CONTAINERD_STEP_VERSIONS array is generated by the server and injected at runtime based on supported containerd versions -CONTAINERD_STEP_VERSIONS=(1.2.13 1.3.9 1.4.13 1.5.11 1.6.22) +CONTAINERD_STEP_VERSIONS=(1.2.13 1.3.9 1.4.13 1.5.11 1.6.24) INSTALLER_YAML= diff --git a/scripts/Manifest b/scripts/Manifest index a0189efb3b..0093566799 100644 --- a/scripts/Manifest +++ b/scripts/Manifest @@ -13,5 +13,5 @@ STEP_VERSIONS=(0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 0.0.0 # ROOK_STEP_VERSIONS array is generated by the server and injected at runtime based on supported rook versions ROOK_STEP_VERSIONS=(1.0.4-14.2.21 0.0.0 0.0.0 0.0.0 1.4.9 1.5.12 1.6.11 1.7.11 1.8.10 1.9.12 1.10.11 1.11.8 1.12.4) # CONTAINERD_STEP_VERSIONS array is generated by the server and injected at runtime based on supported containerd versions -CONTAINERD_STEP_VERSIONS=(1.2.13 1.3.9 1.4.13 1.5.11 1.6.22) +CONTAINERD_STEP_VERSIONS=(1.2.13 1.3.9 1.4.13 1.5.11 1.6.24) INSTALLER_YAML= diff --git a/web/src/installers/versions.js b/web/src/installers/versions.js index 865448f7d0..da9120d502 100644 --- a/web/src/installers/versions.js +++ b/web/src/installers/versions.js @@ -138,7 +138,7 @@ module.exports.InstallerVersions = { "18.09.8", ], containerd: [ - "1.6.22", "1.6.21", "1.6.20", "1.6.19", "1.6.18", "1.6.16", "1.6.15", "1.6.14", "1.6.13", "1.6.12", "1.6.11", "1.6.10", "1.6.9", "1.6.8", "1.6.7", "1.6.6", "1.6.4", "1.5.11", "1.5.10", // cron-containerd-update + "1.6.24", "1.6.22", "1.6.21", "1.6.20", "1.6.19", "1.6.18", "1.6.16", "1.6.15", "1.6.14", "1.6.13", "1.6.12", "1.6.11", "1.6.10", "1.6.9", "1.6.8", "1.6.7", "1.6.6", "1.6.4", "1.5.11", "1.5.10", // cron-containerd-update "1.4.13", "1.4.12", "1.4.11", "1.4.10", "1.4.9", "1.4.8", "1.4.6", "1.4.4", "1.4.3", "1.3.9", "1.3.7", "1.2.13", ], weave: [