enable multinode upgrade tests with weave to flannel (#4839)

* minimize k8s upgrade test * flannel multinode upgrade with sleeps * test with docker to containerd too * sleep if 'yes', prompt otherwise * reconcile base and template * minimize testgrid spec changes
replicatedhq · Sep 26, 2023 · 4702545 · 4702545
1 parent 55212f6
commit 4702545
Show file tree

Hide file tree

Showing 4 changed files with 122 additions and 45 deletions.
diff --git a/addons/flannel/0.22.3/install.sh b/addons/flannel/0.22.3/install.sh
@@ -330,15 +330,15 @@ function flannel_scale_up_rook() {
         logWarn "Failed to scale up Rook Operator within the timeout period."
     fi
 
-    log "Waiting Ceph become healthy"
+    log "Waiting for Ceph to become healthy"
     if ! "$DIR"/bin/kurl rook wait-for-health 1200 ; then
         kubectl logs -n rook-ceph -l app=rook-ceph-operator --all-containers --tail 10 || true
         kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph status
-        logWarn "RookCeph is not healthy. Migration from Weave to Flannel does not seems to be completed successfully"
+        logWarn "Rook Ceph is not healthy. Migration from Weave to Flannel does not seems to have been completed successfully"
         return
     fi
 
-    logSuccess "Rook Ceph is scale up"
+    logSuccess "Rook Ceph scale has been restored"
 }
 
 function weave_to_flannel() {
@@ -348,7 +348,7 @@ function weave_to_flannel() {
     # midair this ensure this function will be ran once more.
     kubectl create configmap -n kurl migration-from-weave --from-literal=started="true" --dry-run=client -o yaml | kubectl apply -f -
 
-    logStep "Scale down services prior remove Weave"
+    logStep "Scaling down services prior to removing Weave"
     flannel_scale_down_ekco
     flannel_scale_down_rook
     flannel_scale_down_prometheus
@@ -387,15 +387,35 @@ function weave_to_flannel() {
         rm /tmp/kotsadm-cert-key
 
         if [ "$AIRGAP" = "1" ]; then
-            printf "\n\t${GREEN}cat ./tasks.sh | sudo bash -s weave-to-flannel-primary airgap cert-key=${cert_key}${NC}\n\n"
+            local command=
+            command=$(printf "cat ./tasks.sh | sudo bash -s weave-to-flannel-primary airgap cert-key=%s" "$cert_key")
+
+            for nodeName in $other_master_nodes; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         else
             local prefix=
             prefix="$(build_installer_prefix "${INSTALLER_ID}" "${KURL_VERSION}" "${KURL_URL}" "${PROXY_ADDRESS}" "${PROXY_HTTPS_ADDRESS}")"
-            printf "\n\t${GREEN}${prefix}tasks.sh | sudo bash -s weave-to-flannel-primary cert-key=${cert_key}${NC}\n\n"
+
+            local command=
+            command=$(printf "%stasks.sh | sudo bash -s weave-to-flannel-primary cert-key=%s" "$prefix" "$cert_key")
+
+            for nodeName in $other_master_nodes; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         fi
 
         printf "${YELLOW}Once this has been run on all nodes, press enter to continue.${NC}"
-        prompt
+        if [ "$ASSUME_YES" = "1" ]; then
+            echo "The 'yes' flag has been passed, so we will wait for 5 minutes here for this to run on remote nodes"
+            sleep 300
+        else
+            prompt
+        fi
 
         logSuccess "User confirmation about nodes execution."
         log "Deleting Pods from kube-flannel"
@@ -414,15 +434,36 @@ function weave_to_flannel() {
         printf "${worker_node_names}\n"
 
         if [ "$AIRGAP" = "1" ]; then
-            printf "\n\t${GREEN}cat ./tasks.sh | sudo bash -s weave-to-flannel-secondary airgap${NC}\n\n"
+            local command=
+            command="cat ./tasks.sh | sudo bash -s weave-to-flannel-secondary airgap"
+
+            for nodeName in $worker_node_names; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         else
             local prefix=
             prefix="$(build_installer_prefix "${INSTALLER_ID}" "${KURL_VERSION}" "${KURL_URL}" "${PROXY_ADDRESS}" "${PROXY_HTTPS_ADDRESS}")"
-            printf "\n\t${GREEN}${prefix}tasks.sh | sudo bash -s weave-to-flannel-secondary${NC}\n\n"
+
+            local command=
+            command=$(printf "%stasks.sh | sudo bash -s weave-to-flannel-secondary" "$prefix")
+
+            for nodeName in $worker_node_names; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         fi
 
         printf "${YELLOW}Once this has been run on all nodes, press enter to continue.${NC}"
-        prompt
+        if [ "$ASSUME_YES" = "1" ]; then
+            echo "The 'yes' flag has been passed, so we will wait for 5 minutes here for this to run on remote nodes"
+            sleep 300
+        else
+            prompt
+        fi
+
         kubectl -n kube-flannel delete pods --all
     fi
 

diff --git a/addons/flannel/template/base/install.sh b/addons/flannel/template/base/install.sh
@@ -330,15 +330,15 @@ function flannel_scale_up_rook() {
         logWarn "Failed to scale up Rook Operator within the timeout period."
     fi
 
-    log "Waiting Ceph become healthy"
+    log "Waiting for Ceph to become healthy"
     if ! "$DIR"/bin/kurl rook wait-for-health 1200 ; then
         kubectl logs -n rook-ceph -l app=rook-ceph-operator --all-containers --tail 10 || true
         kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph status
-        logWarn "RookCeph is not healthy. Migration from Weave to Flannel does not seems to be completed successfully"
+        logWarn "Rook Ceph is not healthy. Migration from Weave to Flannel does not seems to have been completed successfully"
         return
     fi
 
-    logSuccess "Rook Ceph is scale up"
+    logSuccess "Rook Ceph scale has been restored"
 }
 
 function weave_to_flannel() {
@@ -348,7 +348,7 @@ function weave_to_flannel() {
     # midair this ensure this function will be ran once more.
     kubectl create configmap -n kurl migration-from-weave --from-literal=started="true" --dry-run=client -o yaml | kubectl apply -f -
 
-    logStep "Scale down services prior remove Weave"
+    logStep "Scaling down services prior to removing Weave"
     flannel_scale_down_ekco
     flannel_scale_down_rook
     flannel_scale_down_prometheus
@@ -387,15 +387,35 @@ function weave_to_flannel() {
         rm /tmp/kotsadm-cert-key
 
         if [ "$AIRGAP" = "1" ]; then
-            printf "\n\t${GREEN}cat ./tasks.sh | sudo bash -s weave-to-flannel-primary airgap cert-key=${cert_key}${NC}\n\n"
+            local command=
+            command=$(printf "cat ./tasks.sh | sudo bash -s weave-to-flannel-primary airgap cert-key=%s" "$cert_key")
+
+            for nodeName in $other_master_nodes; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         else
             local prefix=
             prefix="$(build_installer_prefix "${INSTALLER_ID}" "${KURL_VERSION}" "${KURL_URL}" "${PROXY_ADDRESS}" "${PROXY_HTTPS_ADDRESS}")"
-            printf "\n\t${GREEN}${prefix}tasks.sh | sudo bash -s weave-to-flannel-primary cert-key=${cert_key}${NC}\n\n"
+
+            local command=
+            command=$(printf "%stasks.sh | sudo bash -s weave-to-flannel-primary cert-key=%s" "$prefix" "$cert_key")
+
+            for nodeName in $other_master_nodes; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         fi
 
         printf "${YELLOW}Once this has been run on all nodes, press enter to continue.${NC}"
-        prompt
+        if [ "$ASSUME_YES" = "1" ]; then
+            echo "The 'yes' flag has been passed, so we will wait for 5 minutes here for this to run on remote nodes"
+            sleep 300
+        else
+            prompt
+        fi
 
         logSuccess "User confirmation about nodes execution."
         log "Deleting Pods from kube-flannel"
@@ -414,15 +434,36 @@ function weave_to_flannel() {
         printf "${worker_node_names}\n"
 
         if [ "$AIRGAP" = "1" ]; then
-            printf "\n\t${GREEN}cat ./tasks.sh | sudo bash -s weave-to-flannel-secondary airgap${NC}\n\n"
+            local command=
+            command="cat ./tasks.sh | sudo bash -s weave-to-flannel-secondary airgap"
+
+            for nodeName in $worker_node_names; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         else
             local prefix=
             prefix="$(build_installer_prefix "${INSTALLER_ID}" "${KURL_VERSION}" "${KURL_URL}" "${PROXY_ADDRESS}" "${PROXY_HTTPS_ADDRESS}")"
-            printf "\n\t${GREEN}${prefix}tasks.sh | sudo bash -s weave-to-flannel-secondary${NC}\n\n"
+
+            local command=
+            command=$(printf "%stasks.sh | sudo bash -s weave-to-flannel-secondary" "$prefix")
+
+            for nodeName in $worker_node_names; do
+                echo "$command" > "$DIR/remotes/$nodeName"
+            done
+
+            printf "\n\t${GREEN}%s${NC}\n\n" "$command"
         fi
 
         printf "${YELLOW}Once this has been run on all nodes, press enter to continue.${NC}"
-        prompt
+        if [ "$ASSUME_YES" = "1" ]; then
+            echo "The 'yes' flag has been passed, so we will wait for 5 minutes here for this to run on remote nodes"
+            sleep 300
+        else
+            prompt
+        fi
+
         kubectl -n kube-flannel delete pods --all
     fi
 

diff --git a/addons/flannel/template/testgrid/k8s-ctrd.yaml b/addons/flannel/template/testgrid/k8s-ctrd.yaml
@@ -65,6 +65,23 @@
     flannel:
       version: "__testver__"
       s3Override: "__testdist__"
+- name: "weave to flannel + docker to containerd multi node"
+  numSecondaryNodes: 2
+  installerSpec:
+    kubernetes:
+      version: "1.22.x"
+    docker:
+      version: "latest"
+    weave:
+      version: "latest"
+  upgradeSpec:
+    kubernetes:
+      version: "1.22.x"
+    containerd:
+      version: "latest"
+    flannel:
+      version: "__testver__"
+      s3Override: "__testdist__"
 - name: "weave to flannel single node with addons and openebs"
   flags: "yes"
   cpu: 6

diff --git a/testgrid/specs/k8s-upgrade.yaml b/testgrid/specs/k8s-upgrade.yaml
@@ -1,4 +1,4 @@
-- name: 119 to 120
+- name: 119 to 128
   numPrimaryNodes: 1
   numSecondaryNodes: 2
   cpu: 3
@@ -12,30 +12,7 @@
       version: latest
   upgradeSpec:
     kubernetes:
-      version: 1.20.x
-    containerd:
-      version: 1.6.x
-    flannel:
-      version: latest
-  postInstallScript: |
-    kubectl get nodes
-  postUpgradeScript: |
-    kubectl get nodes
-- name: 119 to 122
-  numPrimaryNodes: 1
-  numSecondaryNodes: 2
-  cpu: 3
-  flags: "yes"
-  installerSpec:
-    kubernetes:
-      version: 1.19.x
-    containerd:
-      version: 1.6.x
-    flannel:
-      version: latest
-  upgradeSpec:
-    kubernetes:
-      version: 1.22.x
+      version: 1.28.x
     containerd:
       version: 1.6.x
     flannel:
@@ -44,3 +21,4 @@
     kubectl get nodes
   postUpgradeScript: |
     kubectl get nodes
+    kubectl get nodes | grep 1.28