Add retries to docker-compose pull

VirtusLab · Jan 15, 2022 · 05671cf · 05671cf
1 parent 1882d66
commit 05671cf
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 6 deletions.
diff --git a/blogs/docker-ci-tricks-2/blog.md b/blogs/docker-ci-tricks-2/blog.md
@@ -63,11 +63,11 @@ Let's peek into [ci/tox/ci-run.sh](https://github.com/VirtusLab/git-machete/blob
 ```shell script
 # ... skipped ...
 
-source ci/docker-build-and-push.sh tox
+source ci/docker-pull-or-build-and-push.sh tox
 
 # ... skipped ...
 ```
-There is no definition of a `DIRECTORY_HASH` in here, but there is an execution of a [ci/docker-build-and-push.sh](https://github.com/VirtusLab/git-machete/blob/develop/ci/docker-build-and-push.sh) with `tox` argument in which we can see the definition:
+There is no definition of a `DIRECTORY_HASH` in here, but there is an execution of a [ci/docker-pull-or-build-and-push.sh](https://github.com/VirtusLab/git-machete/blob/develop/ci/docker-pull-or-build-and-push.sh) with `tox` argument in which we can see the definition:
 
 ```shell script
 # ... skipped ...

diff --git a/ci/deb-ppa-upload/ci-deploy.sh b/ci/deb-ppa-upload/ci-deploy.sh
@@ -8,7 +8,7 @@ else
   do_dput=true
 fi
 
-source ci/docker-build-and-push.sh deb-ppa-upload
+source ci/docker-pull-or-build-and-push.sh deb-ppa-upload
 
 docker-compose --ansi never run -e TARGET_DISTRO_NAME=bionic -e TARGET_DISTRO_NUMBER=18.04 -e DO_DPUT=$do_dput deb-ppa-upload
 docker-compose --ansi never run -e TARGET_DISTRO_NAME=focal  -e TARGET_DISTRO_NUMBER=20.04 -e DO_DPUT=$do_dput deb-ppa-upload
diff --git a/ci/docker-build-and-push.sh → ci/docker-pull-or-build-and-push.sh b/ci/docker-build-and-push.sh → ci/docker-pull-or-build-and-push.sh
@@ -7,8 +7,26 @@ DIRECTORY_HASH=$(git rev-parse HEAD:ci/$image_name)
 export DIRECTORY_HASH
 cd ci/$image_name/
 
+function retry() {
+  attempts=$1
+  interval=10
+  for i in $(seq 1 $attempts); do
+    if "${@:2}"; then break; fi
+
+    echo "Attempt #$i out of $attempts at command '$*' failed"
+    if [[ $i -lt $attempts ]]; then
+      echo "Sleeping $interval seconds..."
+      sleep $interval
+    fi
+  done
+}
+
 # If image is not found by pull, build the image and push it to the Docker Hub.
-docker-compose --ansi never pull $image_name
+
+# Let's retry pulling the image in case of a spurious failure
+# (`error pulling image configuration: Get "https://docker-images-prod.s3.dualstack.us-east-1.amazonaws.com/...": dial tcp ...:443: i/o timeout`)
+retry 3 docker-compose --ansi never pull $image_name
+
 # A very unpleasant workaround for https://github.com/docker/compose/issues/7258
 # (since v1.25.1, `docker-compose pull` is NOT failing when it can't fetch the image).
 image_tag=$(docker-compose --ansi never config | yq eval ".services.$image_name.image" -)

diff --git a/ci/rpm/ci-run.sh b/ci/rpm/ci-run.sh
@@ -2,6 +2,6 @@
 
 set -e -o pipefail -u -x
 
-source ci/docker-build-and-push.sh rpm
+source ci/docker-pull-or-build-and-push.sh rpm
 
 docker-compose --ansi never run rpm
diff --git a/ci/tox/ci-run.sh b/ci/tox/ci-run.sh
@@ -2,6 +2,6 @@
 
 set -e -o pipefail -u -x
 
-source ci/docker-build-and-push.sh tox
+source ci/docker-pull-or-build-and-push.sh tox
 
 docker-compose --ansi never run tox