retry.sh

#!/bin/bash

# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# retry.sh retries a command until it succeeds. It accepts a regex pattern to match failures on to
# determine if a retry should be attempted.
# Example: retry.sh "connection timed out" ./my-flaky-script.sh some args
# This will run "my-flaky-script.sh", retrying any failed runs that output "connection timed out" up
# to 5 times.

function fail {
  echo "${1}" >&2
  exit 1
}

function isatty() {
 if [ -t 1 ] ; then
   return 0
  else
   return 1
  fi
}

function retry {
  local tmpFile
  tmpFile=$(mktemp)
  trap 'rm -f "${tmpFile}"' EXIT

  local failureRegex="$1"
  shift
  local n=1
  local max=5
  while true; do
    unset SHELL # Don't let environment control which shell to use
    if isatty; then
      if [ "$(uname)" == "Darwin" ]; then
        script -q -r "${tmpFile}" "${*}"
      else
        script --flush --quiet --return "${tmpFile}" --command "${*}"
      fi
    else
      # if we aren't a TTY, run directly as script will always run with a tty, which may output content that
      # we cannot display
      set -o pipefail; "$@" 2>&1 | tee "${tmpFile}"
    fi
    # shellcheck disable=SC2181
    if [[ $? == 0 ]]; then
      break
    fi
    if ! grep -Eq "${failureRegex}" "${tmpFile}"; then
      fail "Unexpected failure"
    fi
    if [[ $n -lt $max ]]; then
      ((n++))
      echo "Command failed. Attempt $n/$max:"
    else
      fail "The command has failed after $n attempts."
    fi
  done
}

retry "$@"