From b182c03407495e23e519bd4289da7733015f32b3 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Thu, 28 Mar 2024 16:14:11 +1100 Subject: [PATCH 1/4] feat: add APPEND_NATIVE_ENTRYPOINT support to lagoon entrypoints --- images/commons/lagoon/entrypoints.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/images/commons/lagoon/entrypoints.sh b/images/commons/lagoon/entrypoints.sh index 55cedfac8..802fa3a01 100755 --- a/images/commons/lagoon/entrypoints.sh +++ b/images/commons/lagoon/entrypoints.sh @@ -12,4 +12,11 @@ if [ -d /lagoon/entrypoints ]; then unset i fi -exec "$@" \ No newline at end of file +# If the image provides a native endpoint that can, or should, be run after the lagoon endpoints are set, it's path can be +# set in the APPEND_NATIVE_ENTRYPOINT variable. +if [ -n "$APPEND_NATIVE_ENTRYPOINT" ] && [ -f $APPEND_NATIVE_ENTRYPOINT ]; then + echo "running defined endpoint" + . $APPEND_NATIVE_ENTRYPOINT +fi + +exec "$@" From cf526950595d5d4b1b52c8f2836545c64ee8d782 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Thu, 28 Mar 2024 16:14:37 +1100 Subject: [PATCH 2/4] feat: implement APPEND_NATIVE_ENTRYPOINT for solr9 --- images/solr-drupal/9.Dockerfile | 1 - images/solr/20-solr-datadir.sh | 156 -------------------------------- images/solr/9.Dockerfile | 8 +- 3 files changed, 3 insertions(+), 162 deletions(-) delete mode 100755 images/solr/20-solr-datadir.sh diff --git a/images/solr-drupal/9.Dockerfile b/images/solr-drupal/9.Dockerfile index 52b000b12..4edab6228 100644 --- a/images/solr-drupal/9.Dockerfile +++ b/images/solr-drupal/9.Dockerfile @@ -11,6 +11,5 @@ LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-image COPY --from=jumpstart /search_api_solr/jump-start/solr9/config-set /solr-conf/conf ENV SOLR_INSTALL_DIR=/opt/solr -RUN mkdir -p /var/solr/data CMD ["solr-precreate", "drupal", "/solr-conf"] diff --git a/images/solr/20-solr-datadir.sh b/images/solr/20-solr-datadir.sh deleted file mode 100755 index 7209f4715..000000000 --- a/images/solr/20-solr-datadir.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/sh -set -eo pipefail - -# check if SOLR_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder -# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container). -# This is different than just setting $SOLR_DATA_DIR to the source folder, as only /var/solr is a persistent folder, so setting -# $SOLR_DATA_DIR to another folder will make solr to not store the datadir across container restarts, while with this copy system -# the data will be prefilled and persistent across container restarts. -if [ -n "$SOLR_COPY_DATA_DIR_SOURCE" ]; then - echo "SOLR_COPY_DATA_DIR_SOURCE is set, start copying from source location" - for solrcorepath in $(ls -d $SOLR_COPY_DATA_DIR_SOURCE/*/ | grep -v lost+found) ; do - corename=$(basename $solrcorepath) - if [ -d ${SOLR_DATA_DIR:-/var/solr}/$corename ]; then - echo "core $corename already present in destination, skipping copying" - else - echo "copying datadir contents from '$SOLR_COPY_DATA_DIR_SOURCE/$corename to '${SOLR_DATA_DIR:-/var/solr}/$corename'" - CUR_DIR=${PWD} - mkdir ${SOLR_DATA_DIR:-/var/solr}/$corename - cd $SOLR_COPY_DATA_DIR_SOURCE/$corename; tar cf - . | (cd ${SOLR_DATA_DIR:-/var/solr}/$corename; tar xvf -) - cd $CUR_DIR - fi - done -fi - -# Previously the Solr Config and Solr Data Dir was both kept in the persistent volume: -# - Solr data: /opt/solr/server/solr/mycores/${corename}/data -# - Solr config: /opt/solr/server/solr/mycores/${corename}/config -# - Persistent Volume: /opt/solr/server/solr/mycores/ -# The Solr Config was copied from the Docker Image into the Solr Config directory on the very first time solr started. -# This had the problem that if a new Solr Config was shipped in a new Docker Iamage the config was not copied again. -# Therefore there was no possibility to ship with updated configs and also the system does not follow any other -# services like nginx or php which have their configs not existing in persistent volumes and insted in Docker Images. -# The following script migrates from to the new directory structure: -# - Solr data: /var/solr/${corename} -# - Solr config: /opt/solr/server/solr/mycores/${corename}/config -# - Persistent Volume: /var/solr/ -# It does: -# 1. Move folders from /var/solr/${corename}/data to /var/solr/${corename} - this is needed if the existing persistent volume is -# mounted now to /var/solr/ but the data is still within data/ -# 2. Create the folder /opt/solr/server/solr/mycores/${corename} if not existing (because there is no persistent volume mounted anymore) -# and copy the config from the persistent storage to that folder. - -# It then also tries to update existing non-compatible configs inside solrconfig.xml: -# - dataDir now needs to be `/var/solr/${solr.core.name}` to point to the new persistent Volume -# - lockType needs to be `${solr.lock.type:none}` to prevent issues with the default file based Lock system which -# can cause issues if the solr is not stopped correctly -# The script does that for existing configs in `/opt/solr/server/solr/mycores/${corename}/config` if that folder exists which can be on two cases: -# 1. During a docker build the solr core has already been created via `precreate-core` (which should be used now all the time) -# 2. The first part of the script has copied the config from the previous persistent volume into these folders -# If `/opt/solr/server/solr/mycores` is empty, this means the container has never been started, had no previous persistent volume and also did not -# run `precreate-core` yet, it checks if the common used folder `/solr-conf/conf/` has a config in it and tries to adapt it. -# This probably fails because of permissions issues, it will throw an error and exit. - -if [ ! -n "$(ls /opt/solr/server/solr/mycores)" ]; then - echo 'No pre-created Solr Cores found in `/opt/solr/server/solr/mycores` this probably means that your Dockerfile does not run' - echo ' RUN precreate-core corename /solr-conf' - echo 'within Dockerfile and instead uses' - echo ' CMD ["solr-precreate", "corename", "/solr-conf"]' - echo 'Please update your Dockerfile to:' - echo ' RUN precreate-core corename /solr-conf' - echo ' CMD ["solr-foreground"]' - printf "\n\n" -fi - -if [ -n "$(ls ${SOLR_DATA_DIR:-/var/solr})" ]; then - # Iterate through all existing solr cores - for solrcorepath in $(ls -d ${SOLR_DATA_DIR:-/var/solr}/*/ | grep -v lost+found) ; do - corename=$(basename $solrcorepath) - if [ -d ${solrcorepath}data ]; then - echo "${solrcorepath} has it's data in deprecated location ${solrcorepath}data, moving to ${solrcorepath}." - # moving the contents of /var/solr/${corename}/data to /var/solr/${corename} - # the datadir now has the layout that a newly created core would. - mv ${solrcorepath}data/* ${solrcorepath} - # remove empty directory - rm -Rf ${solrcorepath}data || mv ${solrcorepath}data ${solrcorepath}data-delete - fi - - # If the core has no files in /opt/solr/server/solr/mycores/${corename} this means: - # The Docker Image did not run `precreate-core corename /solr-conf` during the Dockerfile - # and instead is running `solr-precreate corname solr-conf` as CMD of the container. - # But we already have an existing solr config from the persistent storage, we copy that over - if [ ! -d /opt/solr/server/solr/mycores/${corename} ]; then - mkdir -p /opt/solr/server/solr/mycores/${corename} - # Copy the solr config from the persistent volume in the solr home config directory - cp -R ${solrcorepath}conf /opt/solr/server/solr/mycores/${corename}/ - echo "copied pre-existing solr config from '${solrcorepath}conf' to '/opt/solr/server/solr/mycores/${corename}/conf'" - printf "\n\n" - # there must be a core.properties to be recognized as a core - touch /opt/solr/server/solr/mycores/${corename}/core.properties - fi - done -fi - -function fixConfig { - fail=0 - if cat $1/solrconfig.xml | grep dataDir | grep -qv "${SOLR_DATA_DIR:-/var/solr}/\${solr.core.name}"; then - echo "Found old non lagoon compatible dataDir config in solrconfig.xml:" - cat $1/solrconfig.xml | grep dataDir - SOLR_DATA_DIR=${SOLR_DATA_DIR:-/var/solr} - SOLR_DATA_DIR_ESCAPED=${SOLR_DATA_DIR//\//\\/} # escapig the forward slashes with backslahes - if [ -w $1/ ]; then - sed -ibak "/<\!\-\-/!s/.*/$SOLR_DATA_DIR_ESCAPED\/\${solr.core.name}<\/dataDir>/" $1/solrconfig.xml - echo "automagically updated to compatible config: " - echo " ${SOLR_DATA_DIR:-/var/solr}/\${solr.core.name}" - echo "Please update your solrconfig.xml to make this persistent." - else - echo "but no write permission to automagically change to compatible config: " - echo " ${SOLR_DATA_DIR:-/var/solr}/\${solr.core.name}" - echo "Please update your solrconfig.xml and commit again." - fail=1 - fi - printf "\n\n" - fi - # change lockType to none - if cat $1/solrconfig.xml | grep lockType | grep -qv '${solr.lock.type:none}'; then - echo "Found old non lagoon compatible lockType config in solrconfig.xml:" - cat $1/solrconfig.xml | grep lockType - if [ -w $1/ ]; then - sed -ibak 's/\${solr\.lock\.type:native}<\/lockType>/${solr.lock.type:none}<\/lockType>/' $1/solrconfig.xml - echo "automagically updated to compatible config: " - echo ' ${solr.lock.type:none}' - echo "Please update your solrconfig.xml to make this persistent." - else - echo "but no write permission to automagically change to compatible config: " - echo ' ${solr.lock.type:none}' - echo "Please update your solrconfig.xml and commit again." - fail=1 - fi - printf "\n\n" - fi - if [ "$fail" == "1" ]; then - exit 1; - fi -} - -# check if `/opt/solr/server/solr/mycores` has cores, which means that `precreate-core` has already be called so we check the configs there -if [ -n "$(ls /opt/solr/server/solr/mycores)" ]; then - # Iterate through all solr cores - for solrcorepath in $(ls -d /opt/solr/server/solr/mycores/*/) ; do - corename=$(basename $solrcorepath) - # Check and Update the solr config with lagoon compatible config - if [ -f /opt/solr/server/solr/mycores/${corename}/conf/solrconfig.xml ]; then - fixConfig /opt/solr/server/solr/mycores/${corename}/conf - fi - done -else - # `/opt/solr/server/solr/mycores` is empty, meaning that no `precreate-core` has been called and probably this container is started via `solr-precreate - # We try to update the solr configs within `/solr-conf/conf` to the new lagoon default config as this one will most probably be used to create a new core - if [ -f /solr-conf/conf/solrconfig.xml ]; then - fixConfig /solr-conf/conf - else - echo "No config found in '/solr-conf/conf' and was not able to automatically update solr config to newest lagoon compatible version." - echo "Cannot guarantee if this Solr config will work!" - fi -fi - diff --git a/images/solr/9.Dockerfile b/images/solr/9.Dockerfile index 13694d26c..004f44df0 100755 --- a/images/solr/9.Dockerfile +++ b/images/solr/9.Dockerfile @@ -65,14 +65,12 @@ USER solr ENV SOLR_OPTS="-Dlog4j2.formatMsgNoLookups=true" -COPY 10-solr-port.sh /lagoon/entrypoints/ -# currently, there is no smart upgrade path from 7 to 8 - no autoremediation etc -# and whilst sites may work, upgrading from 7 to 8, they won't work downgrading... -# COPY 20-solr-datadir.sh /lagoon/entrypoints/ - # Define Volume so locally we get persistent cores VOLUME /var/solr +# Define provided solr-docker entrypoint to append +ENV APPEND_NATIVE_ENTRYPOINT=/opt/solr/docker/scripts/docker-entrypoint.sh + ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] CMD ["solr-precreate", "mycore"] From be9675a0e349bba1d894ca7d081953775437be71 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Thu, 28 Mar 2024 16:17:34 +1100 Subject: [PATCH 3/4] test: add solr9 test --- .../TESTING_service_images_dockercompose.md | 18 +++++++++++++++++- helpers/services-docker-compose.yml | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/helpers/TESTING_service_images_dockercompose.md b/helpers/TESTING_service_images_dockercompose.md index 9e445e21b..60724cc5b 100644 --- a/helpers/TESTING_service_images_dockercompose.md +++ b/helpers/TESTING_service_images_dockercompose.md @@ -33,10 +33,12 @@ docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp:/ docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-15:5432 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-16:5432 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mongo-4:27017 -timeout 1m +docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://opensearch-2:9200 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://rabbitmq:15672 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://redis-6:6379 -timeout 1m docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://redis-7:6379 -timeout 1m -docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://opensearch-2:9200 -timeout 1m +docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://solr-8:8983 -timeout 1m +docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://solr-9:8983 -timeout 1m ``` Verification commands @@ -62,6 +64,7 @@ docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep redis-6 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep redis-7 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep solr-8 +docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep solr-9 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep nginx docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep varnish-6 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep varnish-7 @@ -117,6 +120,19 @@ docker-compose exec -T solr-8 sh -c "cat /var/solr/data/mycore/conf/solrconfig.x docker-compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/solr?service=solr-8" | grep "SERVICE_HOST=solr-8" docker-compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/solr?service=solr-8" | grep "LAGOON_TEST_VAR=all-images" +# solr-9 should have a "mycore" Solr core +docker-compose exec -T commons sh -c "curl solr-9:8983/solr/admin/cores?action=STATUS\&core=mycore" + +# solr-9 should be able to reload "mycore" Solr core +docker-compose exec -T commons sh -c "curl solr-9:8983/solr/admin/cores?action=RELOAD\&core=mycore" + +# solr-9 should have solr 9 solrconfig in "mycore" core +docker-compose exec -T solr-9 sh -c "cat /var/solr/data/mycore/conf/solrconfig.xml" | grep luceneMatchVersion | grep 9. + +# solr-9 should be able to read/write data +docker-compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/solr?service=solr-9" | grep "SERVICE_HOST=solr-9" +docker-compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/solr?service=solr-9" | grep "LAGOON_TEST_VAR=all-images" + # mariadb-10-4 should be version 10.4 client docker-compose exec -T mariadb-10-4 sh -c "mysql -V" | grep "10.4" diff --git a/helpers/services-docker-compose.yml b/helpers/services-docker-compose.yml index 1140a1dcb..8b7c1bf7c 100644 --- a/helpers/services-docker-compose.yml +++ b/helpers/services-docker-compose.yml @@ -107,6 +107,12 @@ services: - "8983" user: solr + solr-9: + image: uselagoon/solr-9:latest + ports: + - "8983" + user: solr + varnish-6: image: uselagoon/varnish-6:latest labels: From 4f46870992652b4d4cd7e77849797f3176cb41a6 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 17 Apr 2024 13:03:12 +1000 Subject: [PATCH 4/4] Fix: use correct spelling of entrypoint --- images/commons/lagoon/entrypoints.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/commons/lagoon/entrypoints.sh b/images/commons/lagoon/entrypoints.sh index 802fa3a01..af9738e4c 100755 --- a/images/commons/lagoon/entrypoints.sh +++ b/images/commons/lagoon/entrypoints.sh @@ -12,7 +12,7 @@ if [ -d /lagoon/entrypoints ]; then unset i fi -# If the image provides a native endpoint that can, or should, be run after the lagoon endpoints are set, it's path can be +# If the image provides a native entrypoint that can, or should, be run after the lagoon endpoints are set, it's path can be # set in the APPEND_NATIVE_ENTRYPOINT variable. if [ -n "$APPEND_NATIVE_ENTRYPOINT" ] && [ -f $APPEND_NATIVE_ENTRYPOINT ]; then echo "running defined endpoint"