diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index 2b4b482c91..027b9ce755 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -70,11 +70,15 @@ jobs: - name: provision run: make dev.provision.${{matrix.services}} - - name: dev.up + - name: "Bring up services" run: make dev.up.${{matrix.services}} - - name: dev.check - run: make dev.check.${{matrix.services}} + - name: "Wait for services to become ready" + run: | + # Wait a reasonable amount of time for services to come up. If they + # don't, then call the checks one more time to ensure that diagnostic + # information is printed out. (It's suppressed by wait-for.) + timeout 5m make dev.wait-for.${{matrix.services}} || timeout 1m make dev.check.${{matrix.services}} - name: notify on failure if: ${{ failure() && github.ref == 'refs/heads/master' }} diff --git a/Makefile b/Makefile index 9010b8e853..629da474a3 100644 --- a/Makefile +++ b/Makefile @@ -381,6 +381,9 @@ dev.check: dev.check.$(DEFAULT_SERVICES) ## Run checks for the default service s dev.check.%: # Run checks for a given service or set of services. $(WINPTY) bash ./check.sh $* +dev.wait-for.%: ## Wait for these services to become ready + $(WINPTY) bash ./wait-ready.sh $$(echo $* | tr + " ") + dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE. docker compose config @@ -463,7 +466,7 @@ DB_NAMES_LIST = credentials discovery ecommerce notes registrar xqueue edxapp ed _db_copy8_targets = $(addprefix dev.dbcopy8.,$(DB_NAMES_LIST)) dev.dbcopyall8: ## Copy data from old mysql 5.7 containers into new mysql8 dbs $(MAKE) dev.up.mysql57+mysql80 - ./wait-ready.sh mysql57 mysql80 + $(MAKE) dev.wait-for.mysql57+mysql80 $(MAKE) $(_db_copy8_targets) dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db diff --git a/check.sh b/check.sh index 973302ea88..891cf9b10e 100755 --- a/check.sh +++ b/check.sh @@ -46,7 +46,7 @@ run_check() { if bash -c "$cmd"; then # Run the command itself and check if it succeeded. succeeded="$succeeded $check_name" else - docker compose logs --tail 30 "$service" # Just show recent logs, not all history + docker compose logs --tail 500 "$service" # Just show recent logs, not all history failed="$failed $check_name" fi set -e # Re-enable exit-on-error diff --git a/docs/devstack_interface.rst b/docs/devstack_interface.rst index 5168eec91f..6a24046aef 100644 --- a/docs/devstack_interface.rst +++ b/docs/devstack_interface.rst @@ -200,6 +200,7 @@ If you want to ensure you are getting the latest listing, simply use ``make help dev.up.without-deps.% Bring up services by themselves. dev.up.without-deps.shell.% Bring up a service by itself + shell into it. dev.validate Print effective Docker Compose config, validating files in COMPOSE_FILE. + dev.wait-for.% Wait for these services to become ready devpi-password Get the root devpi password for the devpi container. docs generate Sphinx HTML documentation, including API docs hadoop-application-logs-% View hadoop logs by application Id. diff --git a/provision.sh b/provision.sh index c1e1a4cc75..258abf6e7a 100755 --- a/provision.sh +++ b/provision.sh @@ -130,22 +130,22 @@ fi # Ensure the MySQL5 server is online and usable echo "${GREEN}Waiting for MySQL 5.7.${NC}" -./wait-ready.sh mysql57 +make dev.wait-for.mysql57 # Ensure the MySQL8 server is online and usable echo "${GREEN}Waiting for MySQL 8.0.${NC}" -./wait-ready.sh mysql80 +make dev.wait-for.mysql80 # In the event of a fresh MySQL container, wait a few seconds for the server to restart # See https://github.com/docker-library/mysql/issues/245 for why this is necessary. sleep 10 echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}" -./wait-ready.sh mysql57 +make dev.wait-for.mysql57 echo -e "${GREEN}MySQL5 ready.${NC}" echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}" -./wait-ready.sh mysql80 +make dev.wait-for.mysql80 echo -e "${GREEN}MySQL8 ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. @@ -160,7 +160,7 @@ docker compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql if needs_mongo "$to_provision_ordered"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" # mongo container and mongo process/shell inside the container - ./wait-ready.sh mongo + make dev.wait-for.mongo echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" docker compose exec -T mongo bash -e -c "mongo" < mongo-provision.js