Add error handling, fix some bugs in script

Signed-off-by: Kyle Squizzato <[email protected]>
Mirantis · Oct 13, 2024 · a20a5e0 · a20a5e0
1 parent 8a96b35
commit a20a5e0
Show file tree

Hide file tree

Showing 2 changed files with 102 additions and 33 deletions.
diff --git a/Makefile b/Makefile
@@ -127,7 +127,7 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 add-license: addlicense
 	$(ADDLICENSE) -c "" -ignore ".github/**" -ignore "config/**" -ignore "templates/**" -ignore ".*" -y 2024 .
 
-##@ Build
+##@ Package
 
 TEMPLATES_DIR := templates
 PROVIDER_TEMPLATES_DIR := $(TEMPLATES_DIR)/provider
@@ -142,16 +142,16 @@ $(IMAGES_PACKAGE_DIR): | $(LOCALBIN)
 
 TEMPLATE_FOLDERS = $(patsubst $(TEMPLATES_DIR)/%,%,$(wildcard $(TEMPLATES_DIR)/*))
 
-.PHONY: helm-package ## Package Helm charts used by HMC.
-helm-package: $(CHARTS_PACKAGE_DIR) helm
+.PHONY: helm-package
+helm-package: $(CHARTS_PACKAGE_DIR) helm ## Package Helm charts used by HMC.
 	@make $(patsubst %,package-%-tmpl,$(TEMPLATE_FOLDERS))
 
-.PHONY: bundle-images ## Create a tarball with all images used by HMC.
-bundle-images: $(IMAGES_PACKAGE_DIR)
-	@KUBECTL=$(KUBECTL) YQ=$(YQ) HELM=$(HELM) NAMESPACE=$(NAMESPACE) BUNDLE_TARBALL=$(CHARTS_PACKAGE_DIR)/hmc-images-$(VERSION).tgz bash -c "./scripts/bundle-images.sh"
+.PHONY: bundle-images
+bundle-images: $(IMAGES_PACKAGE_DIR) dev-apply ## Create a tarball with all images used by HMC.
+	@KUBECTL=$(KUBECTL) YQ=$(YQ) HELM=$(HELM) NAMESPACE=$(NAMESPACE) TEMPLATES_DIR=$(TEMPLATES_DIR) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) BUNDLE_TARBALL=$(CHARTS_PACKAGE_DIR)/hmc-images-$(VERSION).tgz bash -c "./scripts/bundle-images.sh"
 
-.PHONY: airgap-package ## Create a tarball with all images and Helm charts used by HMC, useful for deploying in air-gapped environments.
-airgap-package: helm-package bundle-images
+.PHONY: airgap-package
+airgap-package: helm-package bundle-images ## Create a tarball with all images and Helm charts used by HMC, useful for deploying in air-gapped environments.
 	tar -czf hmc-charts-images.tgz $(CHARTS_PACKAGE_DIR) $(IMAGES_PACKAGE_DIR)
 
 package-%-tmpl:
@@ -164,6 +164,8 @@ lint-chart-%:
 package-chart-%: lint-chart-%
 	$(HELM) package --destination $(CHARTS_PACKAGE_DIR) $(TEMPLATES_SUBDIR)/$*
 
+##@ Build
+
 LD_FLAGS?= -s -w
 LD_FLAGS += -X github.com/Mirantis/hmc/internal/build.Version=$(VERSION)
 LD_FLAGS += -X github.com/Mirantis/hmc/internal/telemetry.segmentToken=$(SEGMENT_TOKEN)

diff --git a/scripts/bundle-images.sh b/scripts/bundle-images.sh
@@ -20,51 +20,119 @@
 # this scripts functionality.
 # Usage: make bundle-images
 
-if [ "$(YQ)" == "" ] || [ "$(KUBECTL)" == "" ] || [ "$(HELM)" == "" ] || [ "$(KIND_CLUSTER_NAME)" == "" ] || [ "$(NAMESPACE)" == "" ] || [ "$(BUNDLE_TARBALL)" == "" ]; then
-    echo "This script should not be run directly.  Use 'make bundle-images' instead."
-    exit 1
+set -x
+
+if [ "$YQ" == "" ] ||
+    [ "$KUBECTL" == "" ] ||
+    [ "$HELM" == "" ] ||
+    [ "$KIND_CLUSTER_NAME" == "" ] ||
+    [ "$NAMESPACE" == "" ] ||
+    [ "$BUNDLE_TARBALL" == "" ]
+    [ "$TEMPLATES_DIR" == "" ];
+    then
+        echo "This script should not be run directly.  Use 'make bundle-images' instead."
+        exit 1
 fi
 
 LABEL_KEY="cluster.x-k8s.io/provider"
 IMAGES_BUNDLED=""
 
+wait_for_deploy_exist() {
+    local deployment_label=$1
+    local max_wait_secs=300
+    local interval_secs=5
+    local start_time
+
+
+    start_time=$(date +%s)
+
+    echo "Verifying provider Deployment with label: \"$deployment_label\" exists in namespace: \"$NAMESPACE\"..."
+
+    while true; do
+        current_time=$(date +%s)
+            if (( (current_time - start_time) > max_wait_secs )); then
+                echo "Error: Waited for Deployment with label: \"$deployment_label\" in namespace: \"$NAMESPACE\" to exist for $max_wait_secs seconds and it still does not exist."
+                return 1
+            fi
+
+        output=$($KUBECTL -n "$NAMESPACE" get deploy -l $deployment_label)
+
+        if [[ output != "" ]]; then
+            echo "Deployment in namespace: \"$NAMESPACE\" with label: \"$deployment_label\" exists."
+            break
+        else
+            sleep $interval_secs
+        fi
+    done
+}
+
+echo "Verifying provider Deployments are ready..."
+
 # Verify each provider we support has deployed so we can get the images used
 # across the deployments.
-for TEMPLATE in
-    $(ls templates/provider/*/templates/provider*.yaml)
+for TEMPLATE in $(find $TEMPLATES_DIR -name 'provider.yaml');
 do
     PROVIDER_YAML=$(grep -P '(?=.*Provider)(?=.*kind)' -A 2 $TEMPLATE | grep -v '\--')
-    PROVIDER_KIND=$(echo $PROVIDER | $YQ e '.kind',,)
-    PROVIDER_NAME=$(echo $PROVIDER | $YQ e '.metadata.name')
-    PROVIDER_KIND_TOLOWER=$(echo ${KIND,,})
+    PROVIDER_KIND=$(echo -e "$PROVIDER_YAML" | $YQ e '.kind' -)
+    PROVIDER_NAME=$(echo -e "$PROVIDER_YAML" | $YQ e '.metadata.name' -)
+    PROVIDER_KIND_TOLOWER=$(echo ${PROVIDER_KIND,,})
+
+    if [[ $PROVIDER_NAME == "" ]]; then
+        echo "Error: Cannot determine provider Name from $TEMPLATE"
+        exit 1
+    fi
+
+    if [[ $PROVIDER_KIND_TOLOWER == "" ]]; then
+        echo "Error: Cannot determine provider Kind from $TEMPLATE"
+        exit 1
+    fi
 
     # controlplane is a special case which needs a hyphen.
     if [[ $PROVIDER_KIND_TOLOWER == "controlplane" ]]; then
         PROVIDER_KIND_TOLOWER="control-plane"
     fi
 
-    LABEL_VALUE=$(echo $($PROVIDER_KIND_TOLOWER | sed -e 's/provider//g')-$PROVIDER_NAME)
+    # coreprovider does not have a provider prefix.
+    if [[ $PROVIDER_KIND_TOLOWER == "coreprovider" ]]; then
+        LABEL_VALUE=$(echo $PROVIDER_NAME)
+    else
+        LABEL_VALUE=$(echo $(echo $PROVIDER_KIND_TOLOWER | sed -e 's/provider//g')-$PROVIDER_NAME)
+    fi
 
-    kubectl wait --for condition=available --timeout=2m deploy -l $LABEL_KEY=$LABEL_VALUE -n $NAMESPACE
+    wait_for_deploy_exist "$LABEL_KEY=$LABEL_VALUE"
+
+    $KUBECTL wait --for condition=available --timeout=2m deploy -l $LABEL_KEY=$LABEL_VALUE -n $NAMESPACE
+    if [[ $? -ne 0 ]]; then
+        echo "Error: Cannot wait for Deployment: Deployment with $LABEL_KEY=$LABEL_VALUE label not found"
+        exit 1
+    fi
 done
 
 
 # Now that we know everything is deployed and ready, we can get all of images by
 # execing into the KIND cluster.
-CONTROL_PLANE=$(kubectl get nodes --no-headers -o custom-columns=":metadata.name")
+CONTROL_PLANE=$($KUBECTL get nodes --no-headers -o custom-columns=":metadata.name")
+if [[ $? -ne 0 ]] || [[ $CONTROL_PLANE == "" ]]; then
+    echo "Error: Cannot get control plane node"
+    exit 1
+fi
+
+
+
+for IMAGE in $(docker exec -it $CONTROL_PLANE crictl images | sed 1,1d | awk '{print $1":"$2}' | grep -v "kindest"); do
+    if [[ $IMAGE == "" ]]; then
+        echo "Error: Failed to get image from KIND cluster, image string should not be empty"
+        exit 1
+    fi
 
-for IMAGE in
-    $(docker exec -it $CONTROL_PLANE crictl images | sed 1,1d | awk '{print $1":"$2}' | grep -v "kindest")
-do
     docker pull $IMAGE && docker save $IMAGE -o $BUNDLE_TARBALL_NAME
     echo $IMAGE >> $IMAGES_BUNDLED
 done
 
 
 # Next, we need to build a list of images used by k0s extensions.  Walk the
 # templates directory and extract the images used by the extensions.
-for TEMPLATE in
-    $(find ./templates -name 'k0s*.yaml')
+for TEMPLATE in $(find $TEMPLATES_DIR -name 'k0s*.yaml');
 do
     if [[ $TEMPLATE == *"k0smotron"* ]]; then
         EXTENSIONS_PATH=".spec.k0sConfig.spec.extensions.helm"
@@ -91,7 +159,7 @@ do
         fi
 
         # Use 'helm template' to get the images used by the extension.
-        for IMAGE in $(HELM) template --repo $URL --version $VERSION $NAME | grep -Eo  | cut -d' ' -f2
+        for IMAGE in $(HELM) template --repo $URL --version $VERSION $NAME | grep -Eo | cut -d' ' -f2
         do
             if [[ -z $CUSTOM_TAG ]]; then
                 IMAGE=$(echo $IMAGE | sed -e "s/:.*$/:$CUSTOM_TAG/")
@@ -104,14 +172,13 @@ do
 done
 
 # Cleanup the images bundled by removing them from the local image cache.
-for IMAGE in $IMAGES_BUNDLED
+for IMAGE in $IMAGES_BUNDLED;
 do
-    echo "Removing $IMAGE from local image cache"
+    echo "Removing $IMAGE from local image cache..."
     docker rmi $IMAGE
+    if [ $? -ne 0 ]; then
+        # Note that we failed here but continue trying to remove the other
+        # images.
+        echo "Error: Failed to remove $IMAGE from local image cache"
+    fi
 done
-
-
-
-
-#grep -vw "{{" templates/cluster/aws-hosted-cp/templates/k0smotroncontrolplane.yaml | ./bin/yq-v4.44.2 -e '.spec.k0sConfig.spec.extensions.helm.charts[] | select(.name == "aws-cloud-controller-manager") | .version'
-#grep -vw "{{" templates/cluster/aws-hosted-cp/templates/k0smotroncontrolplane.yaml | ./bin/yq-v4.44.2 -e '.spec.k0sConfig.spec.extensions.helm.charts[] | select(.name == "aws-cloud-controller-manager") | .values' | ./bin/yq-v4.44.2 -e '.image.tag'