Pauld/improve error handling builder (#2243)

* improve the error handling of the builder

* fixes

* improve logging further

* update base builder

* update retry to use = rather than -

builders/container-apps-internal-registry-demo/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_BUILDER_IMAGE="mcr.microsoft.com/oryx/builder:debian-bullseye-20231018.1"
+ARG BASE_BUILDER_IMAGE="mcr.microsoft.com/oryx/builder:debian-bullseye-20231107.1"
 FROM ${BASE_BUILDER_IMAGE}
 
 # these environment variables are generally going to be overwritten:

builders/container-apps-internal-registry-demo/buildCappsBuilder.sh

@@ -12,7 +12,6 @@ source $REPO_DIR/build/__variables.sh
 destinationFqdn="oryxprodmcr.azurecr.io"
 destinationRepo="public/oryx/builder"
 destinationTag="capps-20230208.1"
-baseBuilderImage="mcr.microsoft.com/oryx/builder:debian-bullseye-20231016.1"
 
 PARAMS=""
 while (( "$#" )); do
@@ -29,10 +28,6 @@ while (( "$#" )); do
       destinationTag=$2
       shift 2
       ;;
-    -b|--base-builder-tag)
-      baseBuilderImage=$2
-      shift 2
-      ;;
     --) # end argument parsing
       shift
       break
@@ -60,7 +55,6 @@ echo "Building '$BUILD_IMAGE'..."
 echo
 cd $SCRIPT_DIR
 docker build \
-  --build-arg BASE_BUILDER_IMAGE=$baseBuilderImage \
   -t $BUILD_IMAGE \
   -f Dockerfile \
   .

builders/container-apps-internal-registry-demo/startup-script.sh

@@ -52,84 +52,104 @@ token=$(printf "%s" "$REGISTRY_AUTH_USERNAME:$REGISTRY_AUTH_PASSWORD" | base64)
 acr_access_string="Basic $token"
 export CNB_REGISTRY_AUTH='{"'$ACR_RESOURCE_NAME'":"'$acr_access_string'"}'
 
-echo "Initiating buildpack build..."
-echo "Correlation id: '$CORRELATION_ID'"
+echo "----- Initiating buildpack build -----"
+echo "----- Cloud Build correlation id: '$CORRELATION_ID' -----"
 echo 
 
 RETRY_DELAY=2
 RETRY_ATTEMPTS=5
 
 function fail_if_retry_exceeded() {
   retries=$1
+  exitCode=$2
   if [ "$retries" -ge $RETRY_ATTEMPTS ]; then
     echo "----- Retry attempts exceeded -----"
-    exit 1
+    echo "----- Cloud Build failed with exit code '$lifecycleExitCode' -----"
+    exit $exitCode
   fi
 }
 
+# Allow commands to fail, so we can parse exit codes and handle the failures ourselves.
+set +e
+
 # Execute the analyze phase
 echo
 echo "===== Executing the analyze phase ====="
 retryCount=0
+lifecycleExitCode=0
 until [ "$retryCount" -ge $RETRY_ATTEMPTS ]
 do
   if [ "$retryCount" -ge 1 ]; then
-    echo "----- Retrying analyze phase (attempt $retryCount) -----"
+    echo "===== Retrying analyze phase (attempt $retryCount) ====="
   fi
 
   /lifecycle/analyzer \
     -log-level debug \
     -run-image mcr.microsoft.com/oryx/builder:stack-run-debian-bullseye-20230926.1 \
-    $APP_IMAGE \
-    && break
+    $APP_IMAGE
+
+  lifecycleExitCode=$?
+  if [ "$lifecycleExitCode" -eq 0 ]; then
+    break
+  fi
 
   retryCount=$((retryCount+1))
   sleep $RETRY_DELAY
 done
 
-fail_if_retry_exceeded $retryCount
+fail_if_retry_exceeded $retryCount $lifecycleExitCode
 
 # Execute the detect phase
 echo
 echo "===== Executing the detect phase ====="
 retryCount=0
+lifecycleExitCode=0
 until [ "$retryCount" -ge $RETRY_ATTEMPTS ]
 do
   if [ "$retryCount" -ge 1 ]; then
-    echo "----- Retrying detect phase (attempt $retryCount) -----"
+    echo "===== Retrying detect phase (attempt $retryCount) ====="
   fi
 
   /lifecycle/detector \
     -log-level debug \
-    -app $CNB_APP_DIR \
-    && break
+    -app $CNB_APP_DIR
+
+  lifecycleExitCode=$?
+  if [ "$lifecycleExitCode" -eq 0 ]; then
+    break
+  fi
 
   retryCount=$((retryCount+1))
   sleep $RETRY_DELAY
 done
 
-fail_if_retry_exceeded $retryCount
+fail_if_retry_exceeded $retryCount $lifecycleExitCode
 
 # Execute the restore phase
 echo
 echo "===== Executing the restore phase ====="
 retryCount=0
+lifecycleExitCode=0
 until [ "$retryCount" -ge $RETRY_ATTEMPTS ]
 do
   if [ "$retryCount" -ge 1 ]; then
-    echo "----- Retrying restore phase (attempt $retryCount) -----"
+    echo "===== Retrying restore phase (attempt $retryCount) ====="
   fi
 
   /lifecycle/restorer \
     -log-level debug \
-    -build-image mcr.microsoft.com/oryx/builder:stack-build-debian-bullseye-20230926.1 \
-    && break
+    -build-image mcr.microsoft.com/oryx/builder:stack-build-debian-bullseye-20230926.1
+
+  lifecycleExitCode=$?
+  if [ "$lifecycleExitCode" -eq 0 ]; then
+    break
+  fi
 
   retryCount=$((retryCount+1))
   sleep $RETRY_DELAY
 done
 
-fail_if_retry_exceeded $retryCount
+fail_if_retry_exceeded $retryCount $lifecycleExitCode
 
 # Execute the extend phase
 # Note: we do not retry this, as generally these failures are from the actual build rather than infrastructure.
@@ -140,24 +160,35 @@ echo "===== Executing the extend phase ====="
   -log-level debug \
   -app $CNB_APP_DIR
 
+lifecycleExitCode=$?
+if [ $lifecycleExitCode -ne 0 ]; then
+    echo "----- Cloud Build failed with exit code '$lifecycleExitCode' -----"
+    exit $lifecycleExitCode
+fi
+
 # Execute the export phase
 echo
 echo "===== Executing the export phase ====="
 retryCount=0
+lifecycleExitCode=0
 until [ "$retryCount" -ge $RETRY_ATTEMPTS ]
 do
   if [ "$retryCount" -ge 1 ]; then
-    echo "----- Retrying export phase (attempt $retryCount) -----"
+    echo "===== Retrying export phase (attempt $retryCount) ====="
   fi
 
   /lifecycle/exporter \
     -log-level debug \
     -app $CNB_APP_DIR \
-    $APP_IMAGE \
-    && break
+    $APP_IMAGE
+
+  lifecycleExitCode=$?
+  if [ "$lifecycleExitCode" -eq 0 ]; then
+    break
+  fi
 
   retryCount=$((retryCount+1))
   sleep $RETRY_DELAY
 done
 
-fail_if_retry_exceeded $retryCount
+fail_if_retry_exceeded $retryCount $lifecycleExitCode