Merge branch 'Islandora-Devops:development' into integrahub

.github/workflows/test.yml

@@ -0,0 +1,94 @@
+name: Test
+on:
+  push:
+    paths-ignore:
+      - '**/*.md'
+  schedule:
+    # UTC
+    - cron: '15 12 * * *'
+env:
+  TERM: xterm-256color
+jobs:
+  make:
+    permissions:
+      contents: read
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # TODO: someone with more windows chops please add windows test support
+        # os: [windows-latest, ubuntu-latest, macos-latest]
+        # TODO: keep an eye when macos-14+ (M1) support is available
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Make nightly build check latest isle-buildkit
+        if: github.event_name == 'schedule'
+        run: |-
+          echo "TAG=main" >> $GITHUB_OUTPUT
+
+      - name: Setup make and secrets for Windows
+        if: matrix.os == 'windows-latest'
+        run: |
+          choco install mingw -y
+          echo "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin" >> $GITHUB_PATH
+          cp sample.env .env <-- do not know what windows cp. COPY?
+          C:\Program Files\Git\bin\bash.exe ./build/scripts/check-secrets.sh yes
+
+      - name: init secrets
+        if: matrix.os != 'windows-latest'
+        run: |-
+          cp sample.env .env
+          ./build/scripts/check-secrets.sh yes
+        shell: bash
+
+      - name: make starter
+        run: make starter
+        shell: bash
+
+      - name: check online
+        # TODO: what's a windows curl?
+        if: matrix.os != 'windows-latest'
+        run: |-
+          STATUS=$(curl -k \
+            -w '%{http_code}' -o /dev/null \
+            https://islandora.traefik.me/)
+          echo "Site check returned ${STATUS}"
+          if [ ${STATUS} -ne 200 ]; then
+            echo "Failed to bring up site"
+            exit 1
+          fi
+
+      - name: "Make sure we can export the site config through the UI"
+        run: ./scripts/ci/drush-cex-ui.sh
+
+      - name: make build
+        run: make build
+        shell: bash
+
+      - name: make production
+        run: make production
+        shell: bash
+
+      - name: check online
+        # TODO: what's a windows curl?
+        if: matrix.os != 'windows-latest'
+        run: |-
+          STATUS=$(curl -k \
+            -w '%{http_code}' -o /dev/null \
+            https://islandora.traefik.me/)
+          echo "Site check returned ${STATUS}"
+          if [ ${STATUS} -ne 200 ]; then
+            echo "Failed to bring up site"
+            exit 1
+          fi
+
+      - name: Notify Slack on nightly test failure
+        if: failure() && github.event_name == 'schedule'
+        run: |-
+          curl -s -o /dev/null -XPOST $SLACK_WEBHOOK_URL -d '{
+            "text": "🚨 Scheduled job failed! Click to view the run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Actions Run>",
+          }'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

Makefile

@@ -115,7 +115,9 @@ CMD := $(shell [ $(IS_DRUPAL_PSSWD_FILE_READABLE) -eq 1 ] && echo 'tee' || echo
 LATEST_VERSION := $(shell curl -s https://api.github.com/repos/desandro/masonry/releases/latest | grep '\"tag_name\":' | sed -E 's/.*\"([^\"]+)\".*/\1/')
 
 PHP_FPM_PID=/var/run/php-fpm7/php-fpm7.pid
-ifeq ($(shell expr $(TAG) \>= 3.0), 1)
+ifeq ($(shell expr $(TAG) \>= 3.2), 1)
+	PHP_FPM_PID=/var/run/php-fpm83/php-fpm83.pid
+else ifeq ($(shell expr $(TAG) \>= 3.0), 1)
 	PHP_FPM_PID=/var/run/php-fpm82/php-fpm82.pid
 else ifeq ($(shell expr $(TAG) \>= 2.0), 1)
 	PHP_FPM_PID=/var/run/php-fpm81/php-fpm81.pid
@@ -155,7 +157,7 @@ starter: generate-secrets
 		docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'cd /home/root; composer install'; \
 	fi
 	$(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter
-	docker compose up -d --remove-orphans
+	$(MAKE) compose-up
 	$(MAKE) starter-finalize ENVIRONMENT=starter
 
 
@@ -168,22 +170,14 @@ starter_dev: generate-secrets
 		docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone -b main https://github.com/Islandora-Devops/islandora-starter-site /home/root;'; \
 	fi
 	$(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIRONMENT=starter_dev
-	docker compose up -d --remove-orphans
-		@echo "Wait for the /var/www/drupal directory to be available"
-	while ! docker compose exec -T drupal with-contenv bash -lc 'test -d /var/www/drupal'; do \
-		echo "Waiting for /var/www/drupal directory to be available..."; \
-		sleep 2; \
-	done
+	$(MAKE) compose-up
 	docker compose exec -T drupal with-contenv bash -lc 'chown -R nginx:nginx /var/www/drupal/ ; su nginx -s /bin/bash -c "composer install"'
 	$(MAKE) starter-finalize ENVIRONMENT=starter_dev
 
 
 .PHONY: production
-production: generate-secrets
-	$(MAKE) download-default-certs
-	$(MAKE) -B docker-compose.yml
-	$(MAKE) pull
-	docker compose up -d --remove-orphans
+production: init
+	$(MAKE) compose-up
 	docker compose exec -T drupal with-contenv bash -lc 'composer install; chown -R nginx:nginx .'
 	$(MAKE) drupal-database update-settings-php
 	docker compose exec -T drupal with-contenv bash -lc "drush si -y --existing-config minimal --account-pass '$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)'"
@@ -417,6 +411,28 @@ endif
 
 # Dump fcrepo as zipped tarball
 fcrepo-export:
+ifndef DEST
+	$(error DEST is not set)
+endif
+	docker compose exec -T fcrepo with-contenv bash -lc 'tar zcvf fcrepo-export.tgz -C /data/home/data/ocfl-root/ .'
+	docker compose exec -T fcrepo with-contenv bash -lc 'mv fcrepo-export.tgz /tmp'
+	docker cp $$(docker compose ps -q fcrepo):/tmp/fcrepo-export.tgz $(DEST)
+
+
+# Import fcrepo from zipped tarball
+fcrepo-import: $(SRC)
+ifndef SRC
+	$(error SRC is not set)
+endif
+	docker cp "$(SRC)" $$(docker compose ps -q fcrepo):/tmp/fcrepo-export.tgz
+	docker compose exec -T fcrepo with-contenv bash -lc 'rm -r /data/home/data/ocfl-root/*'
+	docker compose exec -T fcrepo with-contenv bash -lc 'tar zxvf /tmp/fcrepo-export.tgz -C /data/home/data/ocfl-root/ && chown -R tomcat:tomcat /data/home/data/ocfl-root/ && rm /tmp/fcrepo-export.tgz'
+	docker compose exec -T mariadb with-contenv bash -lc 'mysql -e "drop database fcrepo;"'
+	docker compose restart fcrepo
+
+
+# Dump fcrepo as zipped tarball
+fcrepo5-export:
 ifndef DEST
 	$(error DEST is not set)
 endif
@@ -426,7 +442,7 @@ endif
 
 
 # Import fcrepo from zipped tarball
-fcrepo-import: $(SRC)
+fcrepo5-import: $(SRC)
 ifndef SRC
 	$(error SRC is not set)
 endif
@@ -550,14 +566,15 @@ login:
 	docker compose exec -T drupal with-contenv bash -lc "drush uli --uri=$(DOMAIN)"
 	echo "=============================\n"
 
-
-.PHONY: starter-init
-starter-init: generate-secrets
+.PHONY: init
+init: generate-secrets
 	$(MAKE) download-default-certs
 	$(MAKE) -B docker-compose.yml
 	$(MAKE) pull
-	mkdir -p $(CURDIR)/codebase
 
+.PHONY: starter-init
+starter-init: init
+	mkdir -p $(CURDIR)/codebase
 
 .PHONY: starter-finalize
 starter-finalize:
@@ -567,10 +584,8 @@ starter-finalize:
 	docker compose exec -T drupal with-contenv bash -lc "drush -l $(SITE) user:role:add fedoraadmin admin"
 	MIGRATE_IMPORT_USER_OPTION=--userid=1 $(MAKE) hydrate
 	docker compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import --userid=1 --tag=islandora'
-	#docker compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:nginx /var/www/drupal'
-	#docker compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags'
 	$(MAKE) login
-
+	$(MAKE) wait-for-drupal-locally
 
 .PHONY: install
 ## Installs drupal site(s) using environment variables.
@@ -658,4 +673,20 @@ fix_masonry:
 fix_views:
 	docker cp scripts/patch_views.sh $$(docker ps --format "{{.Names}}" | grep drupal):/var/www/drupal/patch_views.sh
 	docker compose exec -T drupal with-contenv bash -lc "bash /var/www/drupal/patch_views.sh ; rm /var/www/drupal/patch_views.sh ; drush cr"
-
+
+.PHONY: compose-up
+.SILENT: compose-up
+compose-up:
+	docker compose up -d --remove-orphans
+	while ! docker compose exec -T drupal with-contenv bash -lc 'test -d /var/www/drupal'; do \
+		echo "Waiting for /var/www/drupal directory to be available..."; \
+		sleep 1; \
+	done
+
+.PHONY: wait-for-drupal-locally
+.SILENT: wait-for-drupal-locally
+wait-for-drupal-locally:
+	while ! curl -s -o /dev/null -m 5 https://$(DOMAIN)/ ; do \
+		echo "Waiting for https://$(DOMAIN) to be available..."; \
+		sleep 1; \
+	done

README.md

@@ -180,7 +180,7 @@ using `CUSTOM_IMAGE_TAG`
 - Run `make push-image` to push that image to your container registry
 
 For convenience a `sample.Dockerfile` is provided which `make build` will use to
-generate a custom image from the [codebase](./codebase) folder. For example if
+generate a custom image from the `codebase` folder. For example if
 you followed the guide above to create the codebase folder from the
 `islandora/demo` image.
 

build/docker-compose/docker-compose.activemq.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   activemq-data:
 services:

build/docker-compose/docker-compose.blazegraph.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   blazegraph-data:
 services:
@@ -24,4 +22,4 @@ services:
           limits:
             memory: ${BLAZEGRAPH_MEMORY_LIMIT:-5G}
           reservations:
-            memory: 2G
+            memory: 2G

build/docker-compose/docker-compose.cantaloupe.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   cantaloupe-data:
 services:

build/docker-compose/docker-compose.code-server.yml

@@ -6,8 +6,6 @@ version: '3.7'
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 secrets:
   CODE_SERVER_PASSWORD:
     file: "../../secrets/live/CODE_SERVER_PASSWORD"

build/docker-compose/docker-compose.crayfish.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 services:
   homarus:
     restart: ${RESTART_POLICY:-unless-stopped}

build/docker-compose/docker-compose.custom.yml

@@ -3,8 +3,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   drupal-sites-data:
   solr-data:

build/docker-compose/docker-compose.demo.yml

@@ -9,8 +9,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   drupal-sites-data:
   solr-data:

build/docker-compose/docker-compose.drupal.yml

@@ -3,8 +3,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 services:
   # The service name is drupal that is the default host name used by micro-services etc.
   # Needs to match against demo, custom, and local.
@@ -20,7 +18,19 @@ services:
       PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT}
       PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE}
       PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE}
+      NGINX_FASTCGI_CONNECT_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_FASTCGI_READ_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_FASTCGI_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_KEEPALIVE_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_PROXY_CONNECT_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_PROXY_READ_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_PROXY_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      NGINX_SEND_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}s
+      PHP_DEFAULT_SOCKET_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}
       PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME}
+      PHP_MAX_INPUT_TIME: ${PHP_MAX_EXECUTION_TIME}
+      PHP_PROCESS_CONTROL_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}
+      PHP_REQUEST_TERMINATE_TIMEOUT: ${PHP_MAX_EXECUTION_TIME}
     labels:
       - traefik.enable=${EXPOSE_DRUPAL:-true}
       - traefik.http.services.${COMPOSE_PROJECT_NAME-isle-dc}-drupal.loadbalancer.server.port=80
@@ -41,4 +51,4 @@ services:
           limits:
             memory: ${DRUPAL_MEMORY_LIMIT:-512M}
           reservations:
-            memory: 256M
+            memory: 256M

build/docker-compose/docker-compose.fcrepo.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   fcrepo-data:
 services:

build/docker-compose/docker-compose.fcrepo6.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   fcrepo-data:
 services:

build/docker-compose/docker-compose.local.yml

@@ -8,8 +8,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   drupal-sites-data:
   solr-data:

build/docker-compose/docker-compose.matomo.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 volumes:
   matomo-config-data:
 services:

build/docker-compose/docker-compose.solr.yml

@@ -2,8 +2,6 @@ version: "3.7"
 networks:
   default:
     internal: true
-  gateway:
-    external: true
 services:
   solr:
     restart: ${RESTART_POLICY:-unless-stopped}