Merge pull request #862 from amazeeio/release/v0.19.2

Release/v0.19.2
uselagoon · Jan 24, 2019 · e827717 · e827717
2 parents a7eb61c + 096fc0f
commit e827717
Show file tree

Hide file tree

Showing 67 changed files with 1,275 additions and 461 deletions.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -290,6 +290,17 @@ services:
       - ./local-dev/api-data-watcher-pusher:/home
     labels:
       lagoon.type: none
+  local-minio:
+    image: minio/minio
+    entrypoint: sh
+    command: -c 'mkdir -p /export/lagoon-files && /usr/bin/minio server /export'
+    ports:
+      - '9000:9000'
+    environment:
+      - MINIO_ACCESS_KEY=minio
+      - MINIO_SECRET_KEY=minio123
+    labels:
+      lagoon.type: none
   drush-alias:
     image: ${IMAGE_REPO:-lagoon}/drush-alias
     volumes:

diff --git a/docs/using_lagoon/index.md b/docs/using_lagoon/index.md
@@ -43,16 +43,17 @@ Some Docker Images and Containers need additional customizations from the provid
 
 | Type           | Versions           | Dockerfile                                                                                                   |
 | ---------------| -------------------| -------------------------------------------------------------------------------------------------------------|
-| nginx | 1.12 | [nginx/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/nginx/Dockerfile) |
+| nginx | openresty/1.13.6.2 | [nginx/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/nginx/Dockerfile) |
 | nginx-drupal | | [nginx-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/nginx-drupal/Dockerfile) |
-| [php-fpm](docker_images/php-fpm.md) | 5.6, 7.0, 7.1, 7.2 | [php/fpm/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/fpm/Dockerfile) |
-| php-cli | 5.6, 7.0, 7.1, 7.2 | [php/cli/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/cli/Dockerfile) |
-| php-cli-drupal | 5.6, 7.0, 7.1, 7.2 | [php/cli-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/cli-drupal/Dockerfile) |
+| [php-fpm](docker_images/php-fpm.md) | 7.1, 7.2, 7.3 | [php/fpm/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/fpm/Dockerfile) |
+| php-cli | 7.1, 7.2, 7.3 | [php/cli/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/cli/Dockerfile) |
+| php-cli-drupal | 7.1, 7.2, 7.3 | [php/cli-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/php/cli-drupal/Dockerfile) |
 | mariadb | 10 | [mariadb/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/mariadb/Dockerfile) |
 | mariadb-drupal | 10 | [mariadb-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/mariadb-drupal/Dockerfile) |
 | mongo | 3.6 | [mongo/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/mongo/Dockerfile) |
-| solr | 5.5, 6.6 | |
-| solr-drupal | 5.5, 6.6 | |
-| redis | | |
-| varnish | 5 | |
-| varnish-drupal | 5 | |
+| solr | 5.5, 6.6, 7.5 | [solr/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/solr-drupal/Dockerfile) |
+| solr-drupal | 5.5, 6.6, 7.5 | [solr-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/solr-drupal/Dockerfile)|
+| redis | 5.0.0 | [redis/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/redis/Dockerfile) |
+| redis-persistent | 5.0.0 | [redis-persistent/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/redis-persistent/Dockerfile) |
+| varnish | 5 | [varnish/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/varnish/Dockerfile) |
+| varnish-drupal | 5 | [varnish-drupal/Dockerfile](https://github.com/amazeeio/lagoon/blob/master/images/varnish-drupal/Dockerfile) |
diff --git a/docs/using_lagoon/remote_shell.md b/docs/using_lagoon/remote_shell.md
@@ -47,7 +47,7 @@ ssh -p [PORT] -t [PROJECT-ENVIRONMENT-NAME]@[HOST] rsh service=[SERVICE-NAME] co
 As example to connect to the `php` container within the `nginx` pod:
 
 ```
-ssh -p 32222 -t [email protected] rsh service=nginx container=pod
+ssh -p 32222 -t [email protected] rsh service=nginx container=php
 ```
 
 ### Execute Commands
@@ -60,4 +60,4 @@ Example:
 ssh -p 32222 -t [email protected] rsh whoami
 ```
 
-Will execute `whoami` within the `cli` container.
+Will execute `whoami` within the `cli` container.
diff --git a/helpers/mariadb-galera2shared.sh b/helpers/mariadb-galera2shared.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+
+
+if [ ! "$1" ]; then
+  echo "please define openshift project as first argument"
+  exit 1;
+fi
+
+set -uo pipefail
+
+which shyaml > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install shyaml (pip3 install shyaml)"
+  exit 1
+fi
+
+which jq > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install jq"
+  exit 1
+fi
+
+which svcat > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install svcat"
+  exit 1
+fi
+
+set -e
+
+PROJECT_NAME=$1
+
+echo "*** Starting mariadb-galera --> mariadb-shared migration in ${PROJECT_NAME}"
+
+SERVICE_NAME=mariadb
+SERVICE_NAME_UPPERCASE=$(echo $SERVICE_NAME | tr [:lower:] [:upper:])
+SERVICE_TYPE=mariadb-shared
+
+ENVIRONMENT_TYPE=$(oc -n $1 get configmap lagoon-env -o json | jq -r '.data.LAGOON_ENVIRONMENT_TYPE')
+
+OLD_POD="mariadb-galera-0"
+
+if [[ "$OLD_POD" ]]; then
+  echo "found $SERVICE_NAME pod $OLD_POD"
+else
+  echo "no running pod found for service '${SERVICE_NAME}'', is it running?"
+  exit 1
+fi
+
+echo "*** Pausing nginx and cli"
+NGINX_REPLICAS=$(oc -n $1 get dc/nginx -o json | jq -r '.spec.replicas')
+CLI_REPLICAS=$(oc -n $1 get dc/cli -o json | jq -r '.spec.replicas')
+oc -n $1 scale dc/nginx --replicas=0
+oc -n $1 scale dc/cli --replicas=0
+
+
+# create service broker
+## taken from build-deploy-docker-compose.sh
+
+OPENSHIFT_TEMPLATE="$(git rev-parse --show-toplevel)/images/oc-build-deploy-dind/openshift-templates/${SERVICE_TYPE}/servicebroker.yml"
+SERVICEBROKER_CLASS="lagoon-dbaas-mariadb-apb"
+SERVICEBROKER_PLAN="${ENVIRONMENT_TYPE}"
+OPENSHIFT_PROJECT=$1
+. $(git rev-parse --show-toplevel)/images/oc-build-deploy-dind/scripts/exec-openshift-create-servicebroker.sh
+
+# ServiceBrokers take a bit, wait until the credentials secret is available
+until oc -n $1 get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials
+do
+  echo "Secret ${SERVICE_NAME}-servicebroker-credentials not available yet, waiting for 10 secs"
+  sleep 10
+done
+
+# Load credentials out of secret
+SECRETS=/tmp/${PROJECT_NAME}-${OLD_POD}-migration.yaml
+oc -n $1 get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials -o yaml > $SECRETS
+
+DB_HOST=$(cat $SECRETS | shyaml get-value data.DB_HOST | base64 -D)
+DB_USER=$(cat $SECRETS | shyaml get-value data.DB_USER | base64 -D)
+DB_PASSWORD=$(cat $SECRETS | shyaml get-value data.DB_PASSWORD | base64 -D)
+DB_NAME=$(cat $SECRETS | shyaml get-value data.DB_NAME | base64 -D)
+DB_PORT=$(cat $SECRETS | shyaml get-value data.DB_PORT | base64 -D)
+
+echo "*** Transfering 'drupal' database from $OLD_POD to $DB_HOST"
+# transfer database between from old to new
+oc -n $1 exec $OLD_POD -- bash -eo pipefail -c "{ mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db drupal || mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -S /tmp/mysql.sock -u \$MYSQL_USER -p\$MYSQL_PASSWORD \$MYSQL_DATABASE; } | sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/' | mysql -h $DB_HOST -u $DB_USER -p${DB_PASSWORD} -P $DB_PORT $DB_NAME"
+
+CONFIG_BAK="/tmp/${PROJECT_NAME}-$(date +%F-%T)-lagoon-env.yaml"
+echo "*** Backing up configmap in case we need to revert: ${CONFIG_BAK}"
+oc -n $1 get configmap lagoon-env -o yaml > $CONFIG_BAK
+
+echo "*** updating configmap to point to ${DB_HOST}."
+# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker
+oc -n $1 patch --insecure-skip-tls-verify configmap lagoon-env \
+   -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}"
+
+
+echo "*** Deleting mariadb service. Scaling old mariadb to 0; you can clean up the DC and pv later"
+oc -n $1 delete service mariadb
+oc -n $1 scale dc/mariadb-maxscale --replicas=0
+oc -n $1 scale statefulset/mariadb-galera --replicas=0
+
+
+# transfer complete, clean up
+rm -f $SECRETS
+
+oc -n $1 scale dc/nginx --replicas=$NGINX_REPLICAS
+oc -n $1 scale dc/cli --replicas=$CLI_REPLICAS
+
+oc -n $1 rollout latest dc/nginx
+oc -n $1 rollout latest dc/cli
+oc -n $1 rollout status dc/nginx
+oc -n $1 rollout status dc/cli
+echo "*** done."
diff --git a/helpers/mariadb-single2shared-no-nginx.sh b/helpers/mariadb-single2shared-no-nginx.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+
+
+if [ ! "$1" ]; then
+  echo "please define openshift project as first argument"
+  exit 1;
+fi
+
+set -uo pipefail
+
+which shyaml > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install shyaml (pip3 install shyaml)"
+  exit 1
+fi
+
+which jq > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install jq"
+  exit 1
+fi
+
+which svcat > /dev/null
+if [ $? -gt 0 ]; then
+  echo "please install svcat"
+  exit 1
+fi
+
+set -e
+
+PROJECT_NAME=$1
+
+echo "*** Starting mariadb-single --> mariadb-shared migration in ${PROJECT_NAME}"
+
+SERVICE_NAME=mariadb
+SERVICE_NAME_UPPERCASE=$(echo $SERVICE_NAME | tr [:lower:] [:upper:])
+SERVICE_TYPE=mariadb-shared
+
+ENVIRONMENT_TYPE=$(oc -n $1 get configmap lagoon-env -o json | jq -r '.data.LAGOON_ENVIRONMENT_TYPE')
+
+MARIADB_REPLICAS=$(oc -n $1 get dc/mariadb -o json | jq -r '.spec.replicas')
+
+if [ "$MARIADB_REPLICAS" == "0" ]; then
+  oc -n $1 scale dc/mariadb --replicas=1
+  oc -n $1 rollout status dc/mariadb
+fi
+
+# export old mariadb pod name
+OLD_POD=$(oc -n $1 get pod -o  custom-columns=NAME:.metadata.name --no-headers -l service=$SERVICE_NAME)
+
+if [[ "$OLD_POD" ]]; then
+  echo "found $SERVICE_NAME pod $OLD_POD"
+else
+  echo "no running pod found for service '${SERVICE_NAME}'', is it running?"
+  exit 1
+fi
+
+# create service broker
+## taken from build-deploy-docker-compose.sh
+
+OPENSHIFT_TEMPLATE="$(git rev-parse --show-toplevel)/images/oc-build-deploy-dind/openshift-templates/${SERVICE_TYPE}/servicebroker.yml"
+SERVICEBROKER_CLASS="lagoon-dbaas-mariadb-apb"
+SERVICEBROKER_PLAN="${ENVIRONMENT_TYPE}"
+OPENSHIFT_PROJECT=$1
+. $(git rev-parse --show-toplevel)/images/oc-build-deploy-dind/scripts/exec-openshift-create-servicebroker.sh
+
+# ServiceBrokers take a bit, wait until the credentials secret is available
+until oc -n $1 get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials
+do
+  echo "Secret ${SERVICE_NAME}-servicebroker-credentials not available yet, waiting for 10 secs"
+  sleep 10
+done
+
+# Load credentials out of secret
+SECRETS=/tmp/${PROJECT_NAME}-${OLD_POD}-migration.yaml
+oc -n $1 get --insecure-skip-tls-verify secret ${SERVICE_NAME}-servicebroker-credentials -o yaml > $SECRETS
+
+DB_HOST=$(cat $SECRETS | shyaml get-value data.DB_HOST | base64 -D)
+DB_USER=$(cat $SECRETS | shyaml get-value data.DB_USER | base64 -D)
+DB_PASSWORD=$(cat $SECRETS | shyaml get-value data.DB_PASSWORD | base64 -D)
+DB_NAME=$(cat $SECRETS | shyaml get-value data.DB_NAME | base64 -D)
+DB_PORT=$(cat $SECRETS | shyaml get-value data.DB_PORT | base64 -D)
+
+echo "*** Transfering 'drupal' database from $OLD_POD to $DB_HOST"
+# transfer database between from old to new
+oc -n $1 exec $OLD_POD -- bash -eo pipefail -c "{ mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db \$MARIADB_DATABASE || mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -S /tmp/mysql.sock -u \$MYSQL_USER -p\$MYSQL_PASSWORD \$MYSQL_DATABASE; } | mysql -h $DB_HOST -u $DB_USER -p${DB_PASSWORD} -P $DB_PORT $DB_NAME"
+
+CONFIG_BAK="/tmp/${PROJECT_NAME}-$(date +%F-%T)-lagoon-env.yaml"
+echo "*** Backing up configmap in case we need to revert: ${CONFIG_BAK}"
+oc -n $1 get configmap lagoon-env -o yaml > $CONFIG_BAK
+
+echo "*** updating configmap to point to ${DB_HOST}."
+# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker
+oc -n $1 patch --insecure-skip-tls-verify configmap lagoon-env \
+   -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}"
+
+
+echo "*** Deleting mariadb service. Scaling old mariadb to 0; you can clean up the DC and pv later"
+oc -n $1 delete service mariadb
+oc -n $1 scale dc/mariadb --replicas=0
+
+# transfer complete, clean up
+rm -f $SECRETS
+
+echo "*** done."