diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..262d11740 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,94 @@ +name: Test +on: + push: + paths-ignore: + - '**/*.md' + schedule: + # UTC + - cron: '15 12 * * *' +env: + TERM: xterm-256color +jobs: + make: + permissions: + contents: read + runs-on: ${{ matrix.os }} + strategy: + matrix: + # TODO: someone with more windows chops please add windows test support + # os: [windows-latest, ubuntu-latest, macos-latest] + # TODO: keep an eye when macos-14+ (M1) support is available + os: [ubuntu-latest] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Make nightly build check latest isle-buildkit + if: github.event_name == 'schedule' + run: |- + echo "TAG=main" >> $GITHUB_OUTPUT + + - name: Setup make and secrets for Windows + if: matrix.os == 'windows-latest' + run: | + choco install mingw -y + echo "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin" >> $GITHUB_PATH + cp sample.env .env <-- do not know what windows cp. COPY? + C:\Program Files\Git\bin\bash.exe ./build/scripts/check-secrets.sh yes + + - name: init secrets + if: matrix.os != 'windows-latest' + run: |- + cp sample.env .env + ./build/scripts/check-secrets.sh yes + shell: bash + + - name: make starter + run: make starter + shell: bash + + - name: check online + # TODO: what's a windows curl? + if: matrix.os != 'windows-latest' + run: |- + STATUS=$(curl -k \ + -w '%{http_code}' -o /dev/null \ + https://islandora.traefik.me/) + echo "Site check returned ${STATUS}" + if [ ${STATUS} -ne 200 ]; then + echo "Failed to bring up site" + exit 1 + fi + + - name: "Make sure we can export the site config through the UI" + run: ./scripts/ci/drush-cex-ui.sh + + - name: make build + run: make build + shell: bash + + - name: make production + run: make production + shell: bash + + - name: check online + # TODO: what's a windows curl? + if: matrix.os != 'windows-latest' + run: |- + STATUS=$(curl -k \ + -w '%{http_code}' -o /dev/null \ + https://islandora.traefik.me/) + echo "Site check returned ${STATUS}" + if [ ${STATUS} -ne 200 ]; then + echo "Failed to bring up site" + exit 1 + fi + + - name: Notify Slack on nightly test failure + if: failure() && github.event_name == 'schedule' + run: |- + curl -s -o /dev/null -XPOST $SLACK_WEBHOOK_URL -d '{ + "text": "🚨 Scheduled job failed! Click to view the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Actions Run>", + }' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/Makefile b/Makefile index 3f8ef1da2..54fe00cd0 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,9 @@ CMD := $(shell [ $(IS_DRUPAL_PSSWD_FILE_READABLE) -eq 1 ] && echo 'tee' || echo LATEST_VERSION := $(shell curl -s https://api.github.com/repos/desandro/masonry/releases/latest | grep '\"tag_name\":' | sed -E 's/.*\"([^\"]+)\".*/\1/') PHP_FPM_PID=/var/run/php-fpm7/php-fpm7.pid -ifeq ($(shell expr $(TAG) \>= 3.0), 1) +ifeq ($(shell expr $(TAG) \>= 3.2), 1) + PHP_FPM_PID=/var/run/php-fpm83/php-fpm83.pid +else ifeq ($(shell expr $(TAG) \>= 3.0), 1) PHP_FPM_PID=/var/run/php-fpm82/php-fpm82.pid else ifeq ($(shell expr $(TAG) \>= 2.0), 1) PHP_FPM_PID=/var/run/php-fpm81/php-fpm81.pid @@ -155,7 +157,7 @@ starter: generate-secrets docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'cd /home/root; composer install'; \ fi $(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter - docker compose up -d --remove-orphans + $(MAKE) compose-up $(MAKE) starter-finalize ENVIRONMENT=starter @@ -168,22 +170,14 @@ starter_dev: generate-secrets docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone -b main https://github.com/Islandora-Devops/islandora-starter-site /home/root;'; \ fi $(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter_dev - docker compose up -d --remove-orphans - @echo "Wait for the /var/www/drupal directory to be available" - while ! docker compose exec -T drupal with-contenv bash -lc 'test -d /var/www/drupal'; do \ - echo "Waiting for /var/www/drupal directory to be available..."; \ - sleep 2; \ - done + $(MAKE) compose-up docker compose exec -T drupal with-contenv bash -lc 'chown -R nginx:nginx /var/www/drupal/ ; su nginx -s /bin/bash -c "composer install"' $(MAKE) starter-finalize ENVIRONMENT=starter_dev .PHONY: production -production: generate-secrets - $(MAKE) download-default-certs - $(MAKE) -B docker-compose.yml - $(MAKE) pull - docker compose up -d --remove-orphans +production: init + $(MAKE) compose-up docker compose exec -T drupal with-contenv bash -lc 'composer install; chown -R nginx:nginx .' $(MAKE) drupal-database update-settings-php docker compose exec -T drupal with-contenv bash -lc "drush si -y --existing-config minimal --account-pass '$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)'" @@ -417,6 +411,28 @@ endif # Dump fcrepo as zipped tarball fcrepo-export: +ifndef DEST + $(error DEST is not set) +endif + docker compose exec -T fcrepo with-contenv bash -lc 'tar zcvf fcrepo-export.tgz -C /data/home/data/ocfl-root/ .' + docker compose exec -T fcrepo with-contenv bash -lc 'mv fcrepo-export.tgz /tmp' + docker cp $$(docker compose ps -q fcrepo):/tmp/fcrepo-export.tgz $(DEST) + + +# Import fcrepo from zipped tarball +fcrepo-import: $(SRC) +ifndef SRC + $(error SRC is not set) +endif + docker cp "$(SRC)" $$(docker compose ps -q fcrepo):/tmp/fcrepo-export.tgz + docker compose exec -T fcrepo with-contenv bash -lc 'rm -r /data/home/data/ocfl-root/*' + docker compose exec -T fcrepo with-contenv bash -lc 'tar zxvf /tmp/fcrepo-export.tgz -C /data/home/data/ocfl-root/ && chown -R tomcat:tomcat /data/home/data/ocfl-root/ && rm /tmp/fcrepo-export.tgz' + docker compose exec -T mariadb with-contenv bash -lc 'mysql -e "drop database fcrepo;"' + docker compose restart fcrepo + + +# Dump fcrepo as zipped tarball +fcrepo5-export: ifndef DEST $(error DEST is not set) endif @@ -426,7 +442,7 @@ endif # Import fcrepo from zipped tarball -fcrepo-import: $(SRC) +fcrepo5-import: $(SRC) ifndef SRC $(error SRC is not set) endif @@ -550,14 +566,15 @@ login: docker compose exec -T drupal with-contenv bash -lc "drush uli --uri=$(DOMAIN)" echo "=============================\n" - -.PHONY: starter-init -starter-init: generate-secrets +.PHONY: init +init: generate-secrets $(MAKE) download-default-certs $(MAKE) -B docker-compose.yml $(MAKE) pull - mkdir -p $(CURDIR)/codebase +.PHONY: starter-init +starter-init: init + mkdir -p $(CURDIR)/codebase .PHONY: starter-finalize starter-finalize: @@ -567,10 +584,8 @@ starter-finalize: docker compose exec -T drupal with-contenv bash -lc "drush -l $(SITE) user:role:add fedoraadmin admin" MIGRATE_IMPORT_USER_OPTION=--userid=1 $(MAKE) hydrate docker compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import --userid=1 --tag=islandora' - #docker compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:nginx /var/www/drupal' - #docker compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags' $(MAKE) login - + $(MAKE) wait-for-drupal-locally .PHONY: install ## Installs drupal site(s) using environment variables. @@ -658,4 +673,20 @@ fix_masonry: fix_views: docker cp scripts/patch_views.sh $$(docker ps --format "{{.Names}}" | grep drupal):/var/www/drupal/patch_views.sh docker compose exec -T drupal with-contenv bash -lc "bash /var/www/drupal/patch_views.sh ; rm /var/www/drupal/patch_views.sh ; drush cr" - + +.PHONY: compose-up +.SILENT: compose-up +compose-up: + docker compose up -d --remove-orphans + while ! docker compose exec -T drupal with-contenv bash -lc 'test -d /var/www/drupal'; do \ + echo "Waiting for /var/www/drupal directory to be available..."; \ + sleep 1; \ + done + +.PHONY: wait-for-drupal-locally +.SILENT: wait-for-drupal-locally +wait-for-drupal-locally: + while ! curl -s -o /dev/null -m 5 https://$(DOMAIN)/ ; do \ + echo "Waiting for https://$(DOMAIN) to be available..."; \ + sleep 1; \ + done diff --git a/README.md b/README.md index 004512e4d..e7a9636d9 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ using `CUSTOM_IMAGE_TAG` - Run `make push-image` to push that image to your container registry For convenience a `sample.Dockerfile` is provided which `make build` will use to -generate a custom image from the [codebase](./codebase) folder. For example if +generate a custom image from the `codebase` folder. For example if you followed the guide above to create the codebase folder from the `islandora/demo` image. diff --git a/build/docker-compose/docker-compose.activemq.yml b/build/docker-compose/docker-compose.activemq.yml index 35f7337fa..edc39cdae 100644 --- a/build/docker-compose/docker-compose.activemq.yml +++ b/build/docker-compose/docker-compose.activemq.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: activemq-data: services: diff --git a/build/docker-compose/docker-compose.blazegraph.yml b/build/docker-compose/docker-compose.blazegraph.yml index 05f733376..b195bf4d4 100644 --- a/build/docker-compose/docker-compose.blazegraph.yml +++ b/build/docker-compose/docker-compose.blazegraph.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: blazegraph-data: services: @@ -24,4 +22,4 @@ services: limits: memory: ${BLAZEGRAPH_MEMORY_LIMIT:-5G} reservations: - memory: 2G \ No newline at end of file + memory: 2G diff --git a/build/docker-compose/docker-compose.cantaloupe.yml b/build/docker-compose/docker-compose.cantaloupe.yml index 3958df2d6..0dd012d7d 100644 --- a/build/docker-compose/docker-compose.cantaloupe.yml +++ b/build/docker-compose/docker-compose.cantaloupe.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: cantaloupe-data: services: diff --git a/build/docker-compose/docker-compose.code-server.yml b/build/docker-compose/docker-compose.code-server.yml index 28528900c..132584a4a 100644 --- a/build/docker-compose/docker-compose.code-server.yml +++ b/build/docker-compose/docker-compose.code-server.yml @@ -6,8 +6,6 @@ version: '3.7' networks: default: internal: true - gateway: - external: true secrets: CODE_SERVER_PASSWORD: file: "../../secrets/live/CODE_SERVER_PASSWORD" diff --git a/build/docker-compose/docker-compose.crayfish.yml b/build/docker-compose/docker-compose.crayfish.yml index d2292c834..16d1b33bf 100644 --- a/build/docker-compose/docker-compose.crayfish.yml +++ b/build/docker-compose/docker-compose.crayfish.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true services: homarus: restart: ${RESTART_POLICY:-unless-stopped} diff --git a/build/docker-compose/docker-compose.custom.yml b/build/docker-compose/docker-compose.custom.yml index 8acc481e9..6896cd707 100644 --- a/build/docker-compose/docker-compose.custom.yml +++ b/build/docker-compose/docker-compose.custom.yml @@ -3,8 +3,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: drupal-sites-data: solr-data: diff --git a/build/docker-compose/docker-compose.demo.yml b/build/docker-compose/docker-compose.demo.yml index 9bd0074fc..d62f6e28b 100644 --- a/build/docker-compose/docker-compose.demo.yml +++ b/build/docker-compose/docker-compose.demo.yml @@ -9,8 +9,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: drupal-sites-data: solr-data: diff --git a/build/docker-compose/docker-compose.drupal.yml b/build/docker-compose/docker-compose.drupal.yml index 52b3f0bea..3e8d670de 100644 --- a/build/docker-compose/docker-compose.drupal.yml +++ b/build/docker-compose/docker-compose.drupal.yml @@ -3,8 +3,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true services: # The service name is drupal that is the default host name used by micro-services etc. # Needs to match against demo, custom, and local. @@ -20,7 +18,19 @@ services: PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT} PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE} PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE} + NGINX_FASTCGI_CONNECT_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_FASTCGI_READ_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_FASTCGI_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_KEEPALIVE_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_PROXY_CONNECT_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_PROXY_READ_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_PROXY_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + NGINX_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s + PHP_DEFAULT_SOCKET_TIMEOUT: ${PHP_MAX_EXECUTION_TIME} PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME} + PHP_MAX_INPUT_TIME: ${PHP_MAX_EXECUTION_TIME} + PHP_PROCESS_CONTROL_TIMEOUT: ${PHP_MAX_EXECUTION_TIME} + PHP_REQUEST_TERMINATE_TIMEOUT: ${PHP_MAX_EXECUTION_TIME} labels: - traefik.enable=${EXPOSE_DRUPAL:-true} - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-drupal.loadbalancer.server.port=80 @@ -41,4 +51,4 @@ services: limits: memory: ${DRUPAL_MEMORY_LIMIT:-512M} reservations: - memory: 256M \ No newline at end of file + memory: 256M diff --git a/build/docker-compose/docker-compose.fcrepo.yml b/build/docker-compose/docker-compose.fcrepo.yml index 74a2ab407..85482deff 100644 --- a/build/docker-compose/docker-compose.fcrepo.yml +++ b/build/docker-compose/docker-compose.fcrepo.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: fcrepo-data: services: diff --git a/build/docker-compose/docker-compose.fcrepo6.yml b/build/docker-compose/docker-compose.fcrepo6.yml index 13d5c820c..67d0a4fad 100644 --- a/build/docker-compose/docker-compose.fcrepo6.yml +++ b/build/docker-compose/docker-compose.fcrepo6.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: fcrepo-data: services: diff --git a/build/docker-compose/docker-compose.local.yml b/build/docker-compose/docker-compose.local.yml index f4f44103d..ed23964c8 100644 --- a/build/docker-compose/docker-compose.local.yml +++ b/build/docker-compose/docker-compose.local.yml @@ -8,8 +8,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: drupal-sites-data: solr-data: diff --git a/build/docker-compose/docker-compose.matomo.yml b/build/docker-compose/docker-compose.matomo.yml index be12f2ddd..962f4a0c3 100644 --- a/build/docker-compose/docker-compose.matomo.yml +++ b/build/docker-compose/docker-compose.matomo.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: matomo-config-data: services: diff --git a/build/docker-compose/docker-compose.solr.yml b/build/docker-compose/docker-compose.solr.yml index c9fa83559..f11f7b844 100644 --- a/build/docker-compose/docker-compose.solr.yml +++ b/build/docker-compose/docker-compose.solr.yml @@ -2,8 +2,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true services: solr: restart: ${RESTART_POLICY:-unless-stopped} diff --git a/build/docker-compose/docker-compose.starter.yml b/build/docker-compose/docker-compose.starter.yml index f4f44103d..ed23964c8 100644 --- a/build/docker-compose/docker-compose.starter.yml +++ b/build/docker-compose/docker-compose.starter.yml @@ -8,8 +8,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: drupal-sites-data: solr-data: diff --git a/build/docker-compose/docker-compose.starter_dev.yml b/build/docker-compose/docker-compose.starter_dev.yml index f4f44103d..ed23964c8 100644 --- a/build/docker-compose/docker-compose.starter_dev.yml +++ b/build/docker-compose/docker-compose.starter_dev.yml @@ -8,8 +8,6 @@ version: "3.7" networks: default: internal: true - gateway: - external: true volumes: drupal-sites-data: solr-data: diff --git a/sample.env b/sample.env index 1023a8a90..3ed7d7afb 100644 --- a/sample.env +++ b/sample.env @@ -1,6 +1,6 @@ # Environment variables defined in this file apply to both the Makefile and to # docker-compose.yml -# +# # Due to restrictions in the `env-file` format we cannot specify multi-line # values for environment variables. For this reason the environment # variables are set on service definitions in the docker-compose.*.yml files, @@ -38,7 +38,7 @@ PROJECT_DRUPAL_DOCKERFILE=Dockerfile CUSTOM_IMAGE_NAMESPACE=mynamespace # Image name of custom drupal image -# This is used when pulling a custom image for environments set to custom and +# This is used when pulling a custom image for environments set to custom and # when building a custom image with make build CUSTOM_IMAGE_NAME=${COMPOSE_PROJECT_NAME}_drupal @@ -49,7 +49,7 @@ CUSTOM_IMAGE_TAG=latest # Packagist repo to use when creating a site with make starter # Change this if you want to build from a different codebase than the starter site -CODEBASE_PACKAGE=islandora/islandora-starter-site:1.2.0 +CODEBASE_PACKAGE=islandora/islandora-starter-site:dev-main # Includes `traefik` as a service, if false assume we are sharing a traefik # from another project. @@ -87,14 +87,14 @@ DRUPAL_DATABASE_SERVICE=mariadb FCREPO_DATABASE_SERVICE=mariadb # Repository to use for pulling isle-buildkit images, change to `local` -# To use images you have built locally with isle-buildkit, or use your +# To use images you have built locally with isle-buildkit, or use your # custom docker registry if you have set up one. # REPOSITORY=islandora # The version of the isle-buildkit images, non isle-buildkit images have # their versions specified explicitly in their respective docker-compose files. -TAG=3.0.0 +TAG=3.2.2 ############################################################################### # Exposed Containers & Ports @@ -155,7 +155,7 @@ RESTART_POLICY=unless-stopped PHP_MEMORY_LIMIT=256M PHP_POST_MAX_SIZE=128M PHP_UPLOAD_MAX_FILESIZE=128M -PHP_MAX_EXECUTION_TIME=30 +PHP_MAX_EXECUTION_TIME=300 # If you're just demoing or are starting from scratch, use this. INSTALL_EXISTING_CONFIG=false diff --git a/scripts/ci/drush-cex-ui.sh b/scripts/ci/drush-cex-ui.sh new file mode 100755 index 000000000..b6abf9991 --- /dev/null +++ b/scripts/ci/drush-cex-ui.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -eou pipefail + +ULI=$(make login | grep traefik) +echo "getting cookie from $ULI" +COOKIE=$(curl -L -s -c - "${ULI}") + +# try exporting the config through the UI +STATUS=$(curl -s \ + --cookie <(echo "$COOKIE") \ + -w '%{http_code}' \ + -o /dev/null \ + https://islandora.traefik.me/admin/config/development/configuration/full/export-download) + +# make sure the config export worked +if [ ${STATUS} -ne 200 ]; then + echo "Could not export config through Drupal UI" + echo ${STATUS} + exit 1 +fi