Add error handling and retry mechanism to curl calls within curlgh

bin/common.sh

@@ -50,5 +50,51 @@ curlgh () {
   else
     skip_verify_arg=""
   fi
-  curl $skip_verify_arg -s -H "Authorization: token $source_access_token" $@
+
+  attempts=0
+  maxAttempts=4
+  sleepDuration=1
+  while [[ $attempts -lt $maxAttempts ]]; do
+    attempts=$((attempts+=1))
+    curl $skip_verify_arg -s -D/tmp/responseheaders -H "Authorization: token $source_access_token" $@ > /tmp/rawresponse
+
+    httpStatus=$(head -n1 /tmp/responseheaders | sed 's|HTTP.* \([0-9]*\) .*|\1|')
+    if [[ "$httpStatus" -eq "200" ]]; then # If HTTP status is OK, skip to extracting the statuses
+      break;
+    fi
+
+    # Various error handling (authn, authz, rate-limiting, transient API errors)
+    if [[ "$httpStatus" -ge 400 ]]; then
+      if [[ "$httpStatus" -lt 500 ]]; then # 4XX range
+        if [[ $(grep -i 'rate-limit' /tmp/rawresponse || echo '0') -ge 1 ]]; then
+          now=$(date "+%s")
+          ratelimitReset=$(cat /tmp/responseheaders | sed -n 's|X-RateLimit-Reset: \([0-9]*\)|\1|p')
+
+          sleepDuration=$((ratelimitReset-now))
+          if [[ "$sleepDuration" -lt 1 ]]; then # Protects against timing issue
+            sleepDuration=1
+          fi
+          echo "Limited by the API rate limit. Script will retry at $(date -d@$((now+sleepDuration)))" >&2
+        else
+          fatal "Authentication error against the GitHub API"
+        fi
+      else # 5XX range
+        echo "Unexpected HTTP $(echo $httpStatus) when querying the GitHub API" >&2
+        sleepDuration=5
+      fi
+    else # Other status code that's not 200 OK, nor in the 400+ range
+      fatal "Unexpected HTTP status code when querying the GitHub API: $(echo $httpStatus)"
+    fi
+
+    # Exit if we have reach the maximum number of attemps, or sleep and retry otherwise
+    if [[ $attempts -eq $maxAttempts ]]; then
+      fatal "Maximum number of attempts reached while trying to query the GitHub API"
+    else
+      echo "Will retry in $sleepDuration seconds" >&2
+      sleep $sleepDuration
+    fi
+
+  done
+
+  cat /tmp/rawresponse
 }

test/in/error-503.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -eu
+
+DIR=$( dirname "$0" )/../..
+
+cat <<EOF | nc -l -s 127.0.0.1 -p 9192 > $TMPDIR/http.req-$$ &
+HTTP/1.0 503 Service Unavailable
+
+EOF
+
+in_dir=$TMPDIR/status-$$
+
+mkdir $in_dir
+
+set +e
+
+$DIR/bin/in "$in_dir" > $TMPDIR/resource-$$ 2>&1 <<EOF
+{
+  "version": {
+    "commit": "6dcb09b5b57875f334f61aebed695e2e4193db5e",
+    "status": "2"
+  },
+  "source": {
+    "access_token": "test-token",
+    "context": "test-context",
+    "endpoint": "http://127.0.0.1:9192",
+    "repository": "dpb587/test-repo"
+  }
+}
+EOF
+
+exitcode=$?
+
+set -e
+
+if ! [ "1" == "$exitcode" ] ; then
+  echo "FAILURE: Expected exit code 1"
+  exit 1
+fi
+
+if ! grep -q 'Status not found on 6dcb09b5b57875f334f61aebed695e2e4193db5e' $TMPDIR/resource-$$ ; then
+  echo "FAILURE: Unexpected failure message"
+  cat $TMPDIR/resource-$$
+  exit 1
+fi