Use substitution in place of cut, wrap vars in {}, fixup airgap pusher

* Only extract the files we need/care about for airgap-push.sh * Use the newly documented directory structure within airgap-push.sh * Ensure full directory paths aren't used in the resulting tarball for 'make airgap-package' * Place the produced airgap bundle from 'make airgap-package' into $LOCALBIN Signed-off-by: Kyle Squizzato <[email protected]>
Mirantis · Nov 4, 2024 · 02ef954 · 02ef954
1 parent c483913
commit 02ef954
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 75 deletions.
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,4 @@ vendor
 mkdocs
 
 # airgap-push script directories
-hmc_charts
+hmc-airgap
diff --git a/Makefile b/Makefile
@@ -156,7 +156,7 @@ bundle-images: dev-apply $(IMAGES_PACKAGE_DIR) ## Create a tarball with all imag
 
 airgap-package: bundle-images ## Create a tarball with all images and Helm charts used by HMC, useful for deploying in air-gapped environments.
 	@TEMPLATES_DIR=$(TEMPLATES_DIR) EXTENSION_CHARTS_PACKAGE_DIR=$(EXTENSION_CHARTS_PACKAGE_DIR) HELM=$(HELM) YQ=$(YQ) $(SHELL) "scripts/package-k0s-extensions-helm.sh"
-	tar -czf hmc-charts-images-$(VERSION).tgz -C $(LOCALBIN) ./scripts/airgap-push.sh $(CHARTS_PACKAGE_DIR) $(IMAGES_PACKAGE_DIR)
+	cd $(LOCALBIN) && tar -czf hmc-airgap-$(VERSION).tgz ../scripts/airgap-push.sh $(shell basename $(CHARTS_PACKAGE_DIR)) $(shell basename $(IMAGES_PACKAGE_DIR))
 
 package-%-tmpl:
 	@make TEMPLATES_SUBDIR=$(TEMPLATES_DIR)/$* $(patsubst %,package-chart-%,$(shell ls $(TEMPLATES_DIR)/$*))

diff --git a/scripts/airgap-push.sh b/scripts/airgap-push.sh
@@ -22,6 +22,10 @@ REPO=""
 CHART_REPO=""
 AIRGAP_BUNDLE=""
 HELP=""
+EXTENSION_TARBALL_PREFIX="hmc-extension-images"
+WORK_DIR="$(pwd)/hmc-airgap"
+
+trap ctrl_c INT
 
 # Parse the options
 while [[ $# -gt 0 ]]; do
@@ -54,7 +58,7 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Print the help message
-print_help() {
+function print_help() {
     echo "Usage:"
     echo "  airgap-push.sh [OPTIONS]"
     echo "Ensure repositories are logged into via 'helm' and 'docker' before running this script."
@@ -69,12 +73,17 @@ print_help() {
     echo "    The path to the airgap bundle"
 }
 
+function ctrl_c() {
+        echo "Caught CTRL-C, exiting..."
+        exit 1
+}
+
 if [ ! -z "$HELP" ]; then
     print_help
     exit 0
 fi
 
-if [ -z "$NEW_REPO" ]; then
+if [ -z "$REPO" ]; then
     echo "The repository must be set"
     print_help
     exit 1
@@ -91,34 +100,72 @@ if [ -z "$AIRGAP_BUNDLE" ]; then
     exit 1
 else
     # Validate the airgap bundle
-    if [ ! -d "$AIRGAP_BUNDLE" ]; then
-        echo "The provided airgap bundle: $AIRGAP_BUNDLE does not exist"
+    if [ ! -f "$AIRGAP_BUNDLE" ]; then
+        echo "The provided airgap bundle: ${AIRGAP_BUNDLE} does not exist"
         exit 1
     fi
 fi
 
-# Load the images into the local Docker daemon.
-docker load -i $AIRGAP_BUNDLE
+mkdir -p ${WORK_DIR}
 
-# Extract the repositories json file from the airgap bundle.
-tar xf $AIRGAP_BUNDLE "repositories"
+# Extract extension images from the airgap bundle.
+echo "Extracting extension images from airgap bundle: ${AIRGAP_BUNDLE}..."
+extension_tarball_name=$(tar tf ${AIRGAP_BUNDLE} | grep "${EXTENSION_TARBALL_PREFIX}")
+tar -C ${WORK_DIR} -xf ${AIRGAP_BUNDLE} ${extension_tarball_name}
+if [ $? -ne 0 ]; then
+    echo "Failed to extract extension images from the airgap bundle"
+    exit 1
+fi
+
+# Load the extension images into the Docker daemon for re-tagging and pushing.
+echo "Loading extension images into Docker..."
+docker load -i ${WORK_DIR}/${extension_tarball_name}
+if [ $? -ne 0 ]; then
+    echo "Failed to load extension images into Docker"
+    exit 1
+fi
 
-for image in $(cat repositories | jq -r 'to_entries[] | .key'); do
-    image_name=$(echo $image | grep -o '[^/]*$')
-    old_image=$(docker images -a | grep $IMAGE | awk '{print $1":"$2}')
-    tag=$old_image | awk -F ":" '{print $2}'
 
-    docker tag $old_image $repo/$image_name:$tag
-    docker push $repo/$image_name:$tag
+# Extract the repositories json file from the extensions bundle.
+echo "Retagging and pushing extension images to ${REPO}..."
+tar -C ${WORK_DIR} -xf ${WORK_DIR}/${extension_tarball_name} "repositories"
+for image in $(cat ${WORK_DIR}/repositories | jq -r 'to_entries[] | .key'); do
+    image_name=$(echo ${image} | grep -o '[^/]*$')
+    old_image=$(docker images -a | grep ${image} | awk '{print $1":"$2}')
+    tag=${old_image#*:}
+    new_image="${REPO}/${image_name}:${tag}"
+
+    docker tag ${old_image} ${new_image}
+    if [ $? -ne 0 ]; then
+        echo "Failed to retag image: ${old_image} with ${new_image}"
+        exit 1
+    fi
+
+    docker push ${new_image}
+    if [ $? -ne 0 ]; then
+        echo "Failed to push image: ${new_image}"
+        exit 1
+    fi
 done
 
-# Next, use Helm to push the charts to the given chart repository.
-mkdir -p hmc_charts
-tar xf $AIRGAP_BUNDLE "charts/extensions" -C hmc_charts/
+# Extract all of the Helm charts from the airgap bundle.
+echo "Extracting Helm charts from the airgap bundle..."
+tar -C ${WORK_DIR} -xf ${AIRGAP_BUNDLE} "charts"
+if [ $? -ne 0 ]; then
+    echo "Failed to extract Helm charts from the airgap bundle"
+    exit 1
+fi
 
-for chart in $(ls hmc_charts/extensions); do
-    helm push hmc_charts/extensions/$chart $CHART_REPO
+# Next, use Helm to push the charts to the given chart repository.
+echo "Pushing Helm charts to ${CHART_REPO}..."
+for chart in $(find ${WORK_DIR}/charts -name "*.tgz"); do
+    helm push ${chart} ${CHART_REPO}
+    if [ $? -ne 0 ]; then
+        echo "Failed to push Helm chart: ${chart}"
+        exit 1
+    fi
 done
 
-# Clean up the extracted files.
-rm -rf hmc_charts
+# Clean up any extracted files.
+echo "Cleaning up..."
+rm -rf ${WORK_DIR}
diff --git a/scripts/bundle-images.sh b/scripts/bundle-images.sh
@@ -36,22 +36,22 @@ function wait_for_deploy_exist() {
 
     start_time=$(date +%s)
 
-    echo "Verifying provider Deployment with label: \"$deployment_label\" exists in namespace: \"$NAMESPACE\"..."
+    echo "Verifying provider Deployment with label: \"${deployment_label}\" exists in namespace: \"${NAMESPACE}\"..."
 
     while true; do
         current_time=$(date +%s)
             if (( (current_time - start_time) > max_wait_secs )); then
-                echo "Error: Waited for Deployment with label: \"$deployment_label\" in namespace: \"$NAMESPACE\" to exist for $max_wait_secs seconds and it still does not exist."
+                echo "Error: Waited for Deployment with label: \"${deployment_label}\" in namespace: \"${NAMESPACE}\" to exist for ${max_wait_secs} seconds and it still does not exist."
                 return 1
             fi
 
-        output=$($KUBECTL -n "$NAMESPACE" get deploy -l $deployment_label)
+        output=$(${KUBECTL} -n "${NAMESPACE}" get deploy -l ${deployment_label})
 
         if [[ $output != "" ]]; then
-            echo "Deployment in namespace: \"$NAMESPACE\" with label: \"$deployment_label\" exists."
+            echo "Deployment in namespace: \"${NAMESPACE}\" with label: \"${deployment_label}\" exists."
             break
         else
-            echo "Deployment with label: \"$deployment_label\" in namespace: \"$NAMESPACE\" does not exist yet.  Waiting $interval_secs seconds..."
+            echo "Deployment with label: \"${deployment_label}\" in namespace: \"${NAMESPACE}\" does not exist yet.  Waiting ${interval_secs} seconds..."
             sleep $interval_secs
         fi
     done
@@ -61,10 +61,10 @@ function bundle_images() {
     local images=$1
     local tarball=$2
 
-    echo "Bundling images into $tarball..."
-    docker save -o $tarball $images
+    echo "Bundling images into ${tarball}..."
+    docker save -o ${tarball} ${images}
     if [[ $? -ne 0 ]]; then
-        echo "Error: Failed to bundle images into $tarball"
+        echo "Error: Failed to bundle images into ${tarball}"
         exit 1
     fi
 }
@@ -79,21 +79,21 @@ echo -e "\nVerifying provider Deployments are ready...\n"
 
 # Verify each provider we support has deployed so we can get the images used
 # across the deployments.
-for template in $(find $TEMPLATES_DIR -name 'provider.yaml');
+for template in $(find ${TEMPLATES_DIR} -name 'provider.yaml');
 do
-    result=$(grep 'kind: .*Provider' $template)
-    provider_yaml=$(grep "$result" -A2 $template)
-    provider_kind=$(echo -e "$provider_yaml" | $YQ e '.kind' -)
-    provider_name=$(echo -e "$provider_yaml" | $YQ e '.metadata.name' -)
+    result=$(grep 'kind: .*Provider' ${template})
+    provider_yaml=$(grep "${result}" -A2 ${template})
+    provider_kind=$(echo -e "${provider_yaml}" | $YQ e '.kind' -)
+    provider_name=$(echo -e "${provider_yaml}" | $YQ e '.metadata.name' -)
     provider_kind_tolower=$(echo ${provider_kind} | tr '[:upper:]' '[:lower:]')
 
     if [[ $provider_name == "" ]]; then
-        echo "Error: Cannot determine provider Name from $template"
+        echo "Error: Cannot determine provider Name from ${template}"
         exit 1
     fi
 
     if [[ $provider_kind_tolower == "" ]]; then
-        echo "Error: Cannot determine provider Kind from $template"
+        echo "Error: Cannot determine provider Kind from ${template}"
         exit 1
     fi
 
@@ -104,32 +104,32 @@ do
 
     # coreprovider does not have a provider prefix.
     if [[ $provider_kind_tolower == "coreprovider" ]]; then
-        label_value=$(echo $provider_name)
+        label_value=$(echo ${provider_name})
     else
-        label_value=$(echo $(echo $provider_kind_tolower | sed -e 's/provider//g')-$provider_name)
+        label_value=$(echo $(echo ${provider_kind_tolower} | sed -e 's/provider//g')-${provider_name})
     fi
 
     wait_for_deploy_exist "$LABEL_KEY=$label_value"
 
-    $KUBECTL wait --for condition=available --timeout=2m deploy -l $LABEL_KEY=$label_value -n $NAMESPACE
+    ${KUBECTL} wait --for condition=available --timeout=2m deploy -l ${LABEL_KEY}=${label_value} -n ${NAMESPACE}
     if [[ $? -ne 0 ]]; then
-        echo "Error: Cannot wait for Deployment: Deployment with $LABEL_KEY=$label_value label not found"
+        echo "Error: Cannot wait for Deployment: Deployment with ${LABEL_KEY}=${label_value} label not found"
         exit 1
     fi
 done
 
 
 # Now that we know everything is deployed and ready, we can get all of images by
 # execing into the KIND cluster.
-control_plane=$($KUBECTL get nodes --no-headers -o custom-columns=":metadata.name")
+control_plane=$(${KUBECTL} get nodes --no-headers -o custom-columns=":metadata.name")
 if [[ $? -ne 0 ]] || [[ $control_plane == "" ]]; then
     echo "Error: Cannot get control plane node"
     exit 1
 fi
 
 echo -e "\nPulling images for HMC components...\n"
 
-for image in $(docker exec -it $control_plane crictl images | sed 1,1d | awk '{print $1":"$2}' | grep -v 'kindest');
+for image in $(docker exec -it ${control_plane} crictl images | sed 1,1d | awk '{print $1":"$2}' | grep -v 'kindest');
 do
     if [[ $image == "" ]]; then
         echo "Error: Failed to get image from KIND cluster, image string should not be empty"
@@ -141,15 +141,15 @@ do
         continue
     fi
 
-    tag=$(echo $image | cut -d':' -f2)
+    tag=${image#*:}
     if [[ $tag == "<none>" ]]; then
-        echo "Will not pull image: $image with tag <none>, continuing..."
+        echo "Will not pull image: ${image} with tag <none>, continuing..."
         continue
     fi
 
     docker pull $image
     if [[ $? -ne 0 ]]; then
-        echo "Error: Failed to pull $image"
+        echo "Error: Failed to pull ${image}"
         exit 1
     fi
 
@@ -160,36 +160,36 @@ echo -e "\nPulling images for HMC extensions...\n"
 
 # Next, we need to build a list of images used by k0s extensions.  Walk the
 # templates directory and extract the images used by the extensions.
-for template in $(find $templates_dir -name 'k0s*.yaml');
+for template in $(find ${templates_dir} -name 'k0s*.yaml');
 do
     if [[ $template == *"k0smotron"* ]]; then
         extensions_path=".spec.k0sConfig.spec.extensions.helm"
     else
         extensions_path=".spec.k0sConfigSpec.k0s.spec.extensions.helm"
     fi
 
-    repos=$(grep -vw "{{" $template | $YQ e "${extensions_path}.repositories[] | [.url, .name] | join(\";\")")
+    repos=$(grep -vw "{{" ${template} | $YQ e "${extensions_path}.repositories[] | [.url, .name] | join(\";\")")
     for repo in $repos
     do
-        url=$(echo $repo | cut -d';' -f1)
-        chartname=$(echo $repo | cut -d';' -f2)
-        version=$(grep -vw "{{" $template |
-            $YQ e "${extensions_path}.charts[] | select(.chartname == \"*$chartname*\") | .version")
-        name=$(grep -vw "{{" $template |
-            $YQ e "${extensions_path}.charts[] | select(.chartname == \"*$chartname*\") | .name")
-        grep -vw "{{" $template | $YQ e "${extensions_path}.charts[] | select(.chartname == \"*$chartname*\") | .values" > $name-values.yaml
+        url=${repo%;*}
+        chartname=${repo#*;}
+        version=$(grep -vw "{{" ${template} |
+            $YQ e "${extensions_path}.charts[] | select(.chartname == \"*${chartname}*\") | .version")
+        name=$(grep -vw "{{" ${template} |
+            $YQ e "${extensions_path}.charts[] | select(.chartname == \"*${chartname}*\") | .name")
+        grep -vw "{{" $template | $YQ e "${extensions_path}.charts[] | select(.chartname == \"*$chartname*\") | .values" > ${name}-values.yaml
 
         if [[ $url == "" ]] || [[ $name == "" ]] || [[ $version == "" ]]; then
-            echo "Error: Failed to get URL, name, or version from $template"
+            echo "Error: Failed to get URL, name, or version from ${template}"
             exit 1
         fi
 
         # Use 'helm template' to get the images used by the extension.
-        for image in $($HELM template --repo $url --version $version $name --values $name-values.yaml | $YQ -N e .spec.template.spec.containers[].image);
+        for image in $(${HELM} template --repo ${url} --version ${version} ${name} --values ${name}-values.yaml | $YQ -N e .spec.template.spec.containers[].image);
         do
-            docker pull $image
+            docker pull ${image}
             if [[ $? -ne 0 ]]; then
-                echo "Error: Failed to pull $image"
+                echo "Error: Failed to pull ${image}"
                 exit 1
             fi
 
@@ -201,11 +201,11 @@ do
 done
 
 echo -e "\nSaving images...\n"
-images_bundled_uniq=$(echo "$IMAGES_BUNDLED" | tr ' ' '\n' | sort -u)
+images_bundled_uniq=$(echo "${IMAGES_BUNDLED}" | tr ' ' '\n' | sort -u)
 bundle_images "$images_bundled_uniq" $BUNDLE_TARBALL
 
 if [[ $EXTENSION_IMAGES_BUNDLED != "" ]]; then
-    extension_images_bundled_uniq=$(echo "$EXTENSION_IMAGES_BUNDLED" | tr ' ' '\n' | sort -u)
+    extension_images_bundled_uniq=$(echo "${EXTENSION_IMAGES_BUNDLED}" | tr ' ' '\n' | sort -u)
     bundle_images "$extension_images_bundled_uniq" $EXTENSIONS_BUNDLE_TARBALL
 fi
 
@@ -214,13 +214,13 @@ echo -e "\nCleaning up all pulled images...\n"
 all_images="$images_bundled_uniq $extension_images_bundled_uniq"
 for image in $all_images;
 do
-    echo "Removing $image from local image cache..."
-    docker rmi $image
+    echo "Removing ${image} from local image cache..."
+    docker rmi ${image}
     if [ $? -ne 0 ]; then
         # Note that we failed here but continue trying to remove the other
         # images.
-        echo "Error: Failed to remove $image from local image cache"
+        echo "Error: Failed to remove ${image} from local image cache"
     fi
 done
 
-echo "Done! Images bundled into $BUNDLE_TARBALL and $EXTENSIONS_BUNDLE_TARBALL"
+echo "Done! Images bundled into ${BUNDLE_TARBALL} and ${EXTENSIONS_BUNDLE_TARBALL}"
diff --git a/scripts/package-k0s-extensions-helm.sh b/scripts/package-k0s-extensions-helm.sh
@@ -15,26 +15,26 @@
 # This script packages Helm charts affiliated with k0s extensions for airgap
 # installations.
 # This script should not be run directly.  Use 'make airgap-package' instead.
-for template in $(find $TEMPLATES_DIR -name 'k0s*.yaml'); do
+for template in $(find ${TEMPLATES_DIR} -name 'k0s*.yaml'); do
     if [[ $template == *"k0smotron"* ]]; then
         extensions_path=".spec.k0sConfig.spec.extensions.helm"
     else
         extensions_path=".spec.k0sConfigSpec.k0s.spec.extensions.helm"
     fi
 
-    repos=$(grep -vw "{{" $template | $YQ e "$extensions_path.repositories[] | [.url, .name] | join(\";\")")
+    repos=$(grep -vw "{{" ${template} | ${YQ} e "${extensions_path}.repositories[] | [.url, .name] | join(\";\")")
     for repo in $repos; do
-            url=$(echo $repo | cut -d';' -f1)
-            chartname=$(echo $repo | cut -d';' -f2)
-            version=$(grep -vw "{{" $template | $YQ e "$extensions_path.charts[] | select(.chartname == \"*$chartname*\") | .version")
-            name=$(grep -vw "{{" $template | $YQ e "${extensions_path}.charts[] | select(.chartname == \"*$chartname*\") | .name")
+            url=${repo%;*}
+            chartname=${repo#*;}
+            version=$(grep -vw "{{" ${template} | $YQ e "${extensions_path}.charts[] | select(.chartname == \"*${chartname}*\") | .version")
+            name=$(grep -vw "{{" ${template} | $YQ e "${extensions_path}.charts[] | select(.chartname == \"*${chartname}*\") | .name")
             if [[ $url == "" ]] || [[ $name == "" ]] || [[ $version == "" ]]; then
                 echo "Error: Cannot construct Helm pull command from url: $url, name: $name, version: $version: one or more vars is not populated"
                 exit 1
             fi
-            if [[ ! $(find $EXTENSION_CHARTS_PACKAGE_DIR -name $name-$version*.tgz) ]]; then
-                echo "Pulling Helm chart $name from $url with version $version"
-                $HELM pull --repo $url --version $version $name -d $EXTENSION_CHARTS_PACKAGE_DIR
+            if [[ ! $(find ${EXTENSION_CHARTS_PACKAGE_DIR} -name ${name}-${version}*.tgz) ]]; then
+                echo "Pulling Helm chart ${name} from ${url} with version ${version}"
+                ${HELM} pull --repo ${url} --version ${version} ${name} -d ${EXTENSION_CHARTS_PACKAGE_DIR}
             fi
     done
 done