attempt overriding containerd pause image on containerd side

addons/containerd/1.6.24/install.sh

@@ -139,9 +139,17 @@ function containerd_configure() {
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = true
 EOF
+    local pause_image=
+    pause_image="$(containerd_kubernetes_pause_image)"
+    if [ -n "$pause_image" ]; then
+        # replace the line 'sandbox_image = "whatever the image previously was"' with 'sandbox_image = "$pause_image"' in /etc/containerd/config.toml
+        sed -i "/sandbox_image/c\\    sandbox_image = \"$pause_image\"" /etc/containerd/config.toml
 
-	  if [ -n "$CONTAINERD_TOML_CONFIG" ]; then
-	      log "Found Containerd TomlConfig set. Installer will patch the value $CONTAINERD_TOML_CONFIG"
+        echo "Set containerd sandbox_image to $pause_image"
+    fi
+
+    if [ -n "$CONTAINERD_TOML_CONFIG" ]; then
+        log "Found Containerd TomlConfig set. Installer will patch the value $CONTAINERD_TOML_CONFIG"
         local tmp=$(mktemp)
         echo "$CONTAINERD_TOML_CONFIG" > "$tmp"
         "$DIR/bin/toml" -basefile=/etc/containerd/config.toml -patchfile="$tmp"
@@ -343,3 +351,16 @@ function _containerd_migrate_images_from_docker() {
         (set -x; ctr -n=k8s.io images import $image)
     done
 }
+
+# return the pause image for the current version of kubernetes
+# versions 1.26 and earlier return the empty string as they can be overridden to use a different image
+function containerd_kubernetes_pause_image() {
+    local minor_version=
+    minor_version="$(kubernetes_version_minor "$KUBERNETES_VERSION")"
+
+    if [ "$minor_version" -ge "27" ]; then
+        cat "$DIR/packages/kubernetes/$KUBERNETES_VERSION/Manifest" | grep "pause" | awk '{ print $3 }'
+    else
+        echo ""
+    fi
+}

addons/containerd/template/base/install.sh

@@ -139,9 +139,17 @@ function containerd_configure() {
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = true
 EOF
+    local pause_image=
+    pause_image="$(containerd_kubernetes_pause_image)"
+    if [ -n "$pause_image" ]; then
+        # replace the line 'sandbox_image = "whatever the image previously was"' with 'sandbox_image = "$pause_image"' in /etc/containerd/config.toml
+        sed -i "/sandbox_image/c\\    sandbox_image = \"$pause_image\"" /etc/containerd/config.toml
 
-	  if [ -n "$CONTAINERD_TOML_CONFIG" ]; then
-	      log "Found Containerd TomlConfig set. Installer will patch the value $CONTAINERD_TOML_CONFIG"
+        echo "Set containerd sandbox_image to $pause_image"
+    fi
+
+    if [ -n "$CONTAINERD_TOML_CONFIG" ]; then
+        log "Found Containerd TomlConfig set. Installer will patch the value $CONTAINERD_TOML_CONFIG"
         local tmp=$(mktemp)
         echo "$CONTAINERD_TOML_CONFIG" > "$tmp"
         "$DIR/bin/toml" -basefile=/etc/containerd/config.toml -patchfile="$tmp"
@@ -258,7 +266,7 @@ function containerd_migrate_from_docker() {
 
     local node=
     node="$(get_local_node_name)"
-    kubectl "$kubeconfigFlag" cordon "$node" 
+    kubectl "$kubeconfigFlag" cordon "$node"
 
     echo "Deleting pods"
     local allPodUIDs=$(kubectl "$kubeconfigFlag" get pods --all-namespaces -ojsonpath='{ range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.namespace}{"\n"}{end}')
@@ -343,3 +351,16 @@ function _containerd_migrate_images_from_docker() {
         (set -x; ctr -n=k8s.io images import $image)
     done
 }
+
+# return the pause image for the current version of kubernetes
+# versions 1.26 and earlier return the empty string as they can be overridden to use a different image
+function containerd_kubernetes_pause_image() {
+    local minor_version=
+    minor_version="$(kubernetes_version_minor "$KUBERNETES_VERSION")"
+
+    if [ "$minor_version" -ge "27" ]; then
+        cat "$DIR/packages/kubernetes/$KUBERNETES_VERSION/Manifest" | grep "pause" | awk '{ print $3 }'
+    else
+        echo ""
+    fi
+}

addons/containerd/template/testgrid/k8s-ctrd.yaml

@@ -1,4 +1,37 @@
-- name: basic containerd and flannel, internal LB
+- name: basic containerd and flannel, internal LB, airgap, 1.28
+  airgap: true
+  installerSpec:
+    kubernetes:
+      version: "1.28.x"
+    flannel:
+      version: latest
+    containerd:
+      version: "__testver__"
+      s3Override: "__testdist__"
+    openebs:
+      version: latest
+      isLocalPVEnabled: true
+      localPVStorageClassName: default
+    minio:
+      version: latest
+    ekco:
+      version: latest
+      enableInternalLoadBalancer: true
+    kotsadm:
+      version: latest
+  preInstallScript: |
+    source /opt/kurl-testgrid/testhelpers.sh
+    rhel_9_install_host_packages lvm2 conntrack-tools socat container-selinux git
+  postInstallScript: |
+    echo "checking /etc/containerd/config.toml"
+    cat /etc/containerd/config.toml | grep pause # should be 3.9, this is for debugging
+    cat /etc/containerd/config.toml | grep pause:3.9
+    echo "checking /var/lib/kubelet/kubeadm-flags.env"
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause # should be 3.9, this is for debugging
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause:3.9
+
+- name: basic containerd and flannel, internal LB, airgap, 1.26 to 1.27, airgap
+  airgap: true
   installerSpec:
     kubernetes:
       version: "1.26.x"
@@ -18,6 +51,44 @@
       enableInternalLoadBalancer: true
     kotsadm:
       version: latest
+  upgradeSpec:
+    kubernetes:
+      version: "1.27.x"
+    flannel:
+      version: latest
+    containerd:
+      version: "__testver__"
+      s3Override: "__testdist__"
+    openebs:
+      version: latest
+      isLocalPVEnabled: true
+      localPVStorageClassName: default
+    minio:
+      version: latest
+    ekco:
+      version: latest
+      enableInternalLoadBalancer: true
+    kotsadm:
+      version: latest
+  preInstallScript: |
+    source /opt/kurl-testgrid/testhelpers.sh
+    rhel_9_install_host_packages lvm2 conntrack-tools socat container-selinux git
+  postInstallScript: |
+    echo "checking /etc/containerd/config.toml"
+    cat /etc/containerd/config.toml | grep pause # should be 3.6, this is for debugging
+    cat /etc/containerd/config.toml | grep pause:3.6 
+    echo "checking /var/lib/kubelet/kubeadm-flags.env"
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause # should be 3.6, this is for debugging
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause:3.6
+  postUpgradeScript: |
+    echo "checking /etc/containerd/config.toml"
+    cat /etc/containerd/config.toml | grep pause # should be 3.9, this is for debugging
+    cat /etc/containerd/config.toml | grep pause:3.9 
+    echo "checking /var/lib/kubelet/kubeadm-flags.env"
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause # should be 3.9, this is for debugging
+    cat /var/lib/kubelet/kubeadm-flags.env | grep pause:3.9
+
+
 
 - name: "Migrate from Docker to Containerd and Kubernetes from 1.23 to 1.25"
   flags: "yes"