diff --git a/Jenkinsfile b/Jenkinsfile index 085dbe9164..18b234fa85 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,7 +34,7 @@ node { }, 'start minishift': { stage ('start minishift') { - sh 'make minishift' + sh 'make minishift MINISHIFT_CPUS=8 MINISHIFT_MEMORY=12GB MINISHIFT_DISK_SIZE=50GB' } } ) @@ -49,7 +49,7 @@ node { stage ('run tests') { try { sh "make push-minishift" - sh "make tests -j4" + sh "make tests -j5" } catch (e) { echo "Something went wrong, trying to cleanup" cleanup() diff --git a/Makefile b/Makefile index a5fdc546a2..247a349e7f 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,11 @@ SHELL := /bin/bash DOCKER_BUILD_PARAMS := --quiet # Version and Hash of the OpenShift cli that should be downloaded -MINISHIFT_VERSION := 1.9.0 +MINISHIFT_VERSION := 1.15.1 + +MINISHIFT_CPUS := 6 +MINISHIFT_MEMORY := 2GB +MINISHIFT_DISK_SIZE := 30GB # On CI systems like jenkins we need a way to run multiple testings at the same time. We expect the # CI systems to define an Environment variable CI_BUILD_TAG which uniquely identifies each build. @@ -327,6 +331,7 @@ build/logs2logs-db: build/logstash build/logs-db: build/elasticsearch build/logs-db-ui: build/kibana build/auto-idler: build/oc +build/api-db: build/mariadb # Auth SSH needs the context of the root folder, so we have it individually build/ssh: build/commons @@ -384,6 +389,8 @@ build-list: all-tests-list:= features \ node \ drupal \ + drupal-postgres \ + drupal-galera \ github \ gitlab \ bitbucket \ @@ -417,7 +424,7 @@ $(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/o IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter) -tests/drupal: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift +tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift $(eval testname = $(subst tests/,,$@)) IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) drush-alias IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter) @@ -572,7 +579,7 @@ openshift: # that has been assigned to the machine is not the default one and then replace the IP in the yaml files with it minishift: local-dev/minishift/minishift $(info starting minishift with name $(CI_BUILD_TAG)) - ./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) start --cpus 6 --vm-driver virtualbox --openshift-version="v3.6.1" + ./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) start --cpus $(MINISHIFT_CPUS) --memory $(MINISHIFT_MEMORY) --disk-size $(MINISHIFT_DISK_SIZE) --vm-driver virtualbox --openshift-version="v3.7.2" ifeq ($(ARCH), Darwin) @OPENSHIFT_MACHINE_IP=$$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) ip); \ echo "replacing IP in local-dev/api-data/api-data.gql and docker-compose.yaml with the IP '$$OPENSHIFT_MACHINE_IP'"; \ @@ -610,7 +617,7 @@ openshift-lagoon-setup: oc -n lagoon create -f openshift-setup/clusterrole-openshiftbuilddeploy.yaml; \ oc -n lagoon adm policy add-cluster-role-to-user openshiftbuilddeploy -z openshiftbuilddeploy; \ oc -n lagoon create -f openshift-setup/shared-resource-viewer.yaml; \ - oc -n lagoon create -f openshift-setup/policybinding.yaml; \ + oc -n lagoon create -f openshift-setup/policybinding.yaml | oc -n lagoon create -f openshift-setup/rolebinding.yaml; \ oc -n lagoon create serviceaccount docker-host; \ oc -n lagoon adm policy add-scc-to-user privileged -z docker-host; \ oc -n lagoon policy add-role-to-user edit -z docker-host; \ diff --git a/docs/using_lagoon/drupal/lagoonize.md b/docs/using_lagoon/drupal/lagoonize.md index e94c9aafba..fdf77071bc 100644 --- a/docs/using_lagoon/drupal/lagoonize.md +++ b/docs/using_lagoon/drupal/lagoonize.md @@ -7,7 +7,7 @@ In order for Drupal to work with Lagoon we need to teach Drupal about Lagoon and You find these Files [here](https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal). Probably easiest is to download them as [ZIP File](https://minhaskamal.github.io/DownGit/#/home?url=https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal) and copy them into your Git Repository. For each Drupal Version and Database Type you will find an individual folder. A short overview of what they are: - `.lagoon.yml` - They main file that will be used by Lagoon to understand what should be deployed and many more things. This file has some sensible Drupal defaults, if you like to edit or modify, please check the specific [Documentation for .lagoon.yml]() -- `docker-compose.yml`, `.dockerignore` and `Dockerfile.*` - These files are used to run your Local Drupal Development enviornment, it tells docker which services to start and how to build them. These contain sensible defaults and many commented lines, it should be pretty much self describing. If you like more, see [Documentation for docker-compose.yml]() +- `docker-compose.yml`, `.dockerignore` and `Dockerfile.*` - These files are used to run your Local Drupal Development environment, it tells docker which services to start and how to build them. These contain sensible defaults and many commented lines, it should be pretty much self describing. If you like more, see [Documentation for docker-compose.yml]() - `sites/default/*` - These .php and .yml files teach Drupal how to communicate with Lagoon containers and this locally and in production. It also provides an easy system for specific overrides in development and production environments. Unlike other Drupal Hosting Systems, Lagoon never ever injects Drupal Settings files into your Drupal. Therefore you can edit them to your wish. Like all other files they contain sensible defaults and some commented parts. - `drush/alises.drushrc.php` - These files are specific to Drush and tell Drush how to talk to the Lagoon GraphQL API in order to learn about all Site Aliases there are. - `drush/drushrc.php` - Some sensible defaults for Drush Commands. @@ -54,7 +54,7 @@ This maybe sounds weird, as there was already a `composer install` executed duri ## 5. Check Status and Install Drupal -Finally it's time to install a Drupal, but just before that we want to make sure everything works allright. We sugget to use Drush for that: +Finally it's time to install a Drupal, but just before that we want to make sure everything works alright. We suggest to use Drush for that: docker-compose exec cli bash drush status @@ -147,4 +147,4 @@ A Drupal Site also consists of the files directory. As the whole folder is mount You are done. The Lagoon Team wishes Happy Drupaling! -If you like to deploy your Drupal into Lagoon, follow the next Step: [Setup a new Project in Lagoon](../setup_project.md) \ No newline at end of file +If you like to deploy your Drupal into Lagoon, follow the next Step: [Setup a new Project in Lagoon](../setup_project.md) diff --git a/docs/using_lagoon/first_deployment.md b/docs/using_lagoon/first_deployment.md index ed52593e16..a2479b92e3 100644 --- a/docs/using_lagoon/first_deployment.md +++ b/docs/using_lagoon/first_deployment.md @@ -8,7 +8,7 @@ Also a short Heads-Up: If you are deploying a Drupal Project we suggest to read ## 1. Make sure you are all set -In order to make your first deployment a successfull one, please make sure that your Project is Lagoonized and you have setup the Project in Lagoon. If not, don't worry, just follow the [Step-by-Step Guides](./index.md) which show you how this works. +In order to make your first deployment a successful one, please make sure that your Project is Lagoonized and you have setup the Project in Lagoon. If not, don't worry, just follow the [Step-by-Step Guides](./index.md) which show you how this works. ## 2. Push! @@ -51,4 +51,4 @@ Did the deployment fail? Oh no! But we're here to help: 1. If you deployed a Drupal site, probably best read the [Drupal specific First Deployment Documentation](./drupal/first_deployment.md) which explains why this happens. 2. Click on the `Logs` Link in the Error Notification, it will tell you where in the Deployment Process the failure happened. -3. If you can't figure it out, just ask your Lagoon Support, they are definitely here to help! \ No newline at end of file +3. If you can't figure it out, just ask your Lagoon Support, they are definitely here to help! diff --git a/docs/using_lagoon/index.md b/docs/using_lagoon/index.md index b866315452..eb8cfebc6e 100644 --- a/docs/using_lagoon/index.md +++ b/docs/using_lagoon/index.md @@ -36,7 +36,7 @@ This file is used by Docker Compose to start you Local Development environment. Some Docker Images and Containers need additional customizations from the provided Images, this usually has two reasons: 1. Application code: Containers like Nginx, PHP, Node, etc. need the actual programming code within their Images. This is done during a Docker Build step which are configured in Dockerfiles. Lagoon has full support for Docker and therefore also allows you full control over the resulting Images via Dockerfile customizations. -2. Customization of Images: Lagoon also allows you to customize the base Images accordint to your needs. This can be to inject an addtional environment variable, change a service configuration or even install additional tools, even though we advise caution with installing additional tools to the Docker Images, as such adaptions need to be maintained by yourself in the future! +2. Customization of Images: Lagoon also allows you to customize the base Images according to your needs. This can be to inject an addtional environment variable, change a service configuration or even install additional tools, even though we advise caution with installing additional tools to the Docker Images, as such adaptions need to be maintained by yourself in the future! ## Supported Services & Base Images by Lagoon diff --git a/docs/using_lagoon/setup_project.md b/docs/using_lagoon/setup_project.md index e5cc7aff3f..b284e5dfe7 100644 --- a/docs/using_lagoon/setup_project.md +++ b/docs/using_lagoon/setup_project.md @@ -3,7 +3,7 @@ **NOTE: We are working hard on having a CLI and GraphQL API ready for everybody to setup and configure their projects themselves. It needs a couple of more days testing.** Until then the setup of a new project involves talking to your Lagoon Administrator. They are anyway much friendlier then APIs ;) -Please have the following infos ready for your Lagoon Administrator: +Please have the following information ready for your Lagoon Administrator: - SSH-Public Keys of everybody that will work on this project - The URL of the Git repository which will host the code (git@example.com:test/test.git) - The Git branch you would like to use as your production environment @@ -19,7 +19,7 @@ If this is not the case, check out the list of [Step-by-Step Guides](./index.md) In order to deploy your Code, Lagoon needs access to it. By design and for security Lagoon only needs read access to your Git Repository. -Your Lagon Administrator will tell you the SSH Public Key or the Git Account to give read access to. +Your Lagoon Administrator will tell you the SSH Public Key or the Git Account to give read access to. ## 3. Configure Webhook diff --git a/images/oc-build-deploy-dind/Dockerfile b/images/oc-build-deploy-dind/Dockerfile index b6e98c30f3..bdbae58ad1 100644 --- a/images/oc-build-deploy-dind/Dockerfile +++ b/images/oc-build-deploy-dind/Dockerfile @@ -5,16 +5,20 @@ FROM ${IMAGE_REPO:-lagoon}/oc RUN rm -rf /root && ln -s /home /root ENV LAGOON=oc-build-deploy-dind -RUN mkdir -p /git +RUN mkdir -p /oc-build-deploy/git +RUN mkdir -p /oc-build-deploy/tug +RUN mkdir -p /oc-build-deploy/lagoon -WORKDIR /git +WORKDIR /oc-build-deploy/git COPY docker-entrypoint.sh /lagoon/entrypoints/100-docker-entrypoint.sh -COPY build-deploy.sh /usr/sbin/build-deploy -COPY build-deploy-docker-compose.sh /build-deploy-docker-compose.sh +COPY build-deploy.sh /oc-build-deploy/build-deploy.sh +COPY build-deploy-docker-compose.sh /oc-build-deploy/build-deploy-docker-compose.sh +COPY tug.sh /oc-build-deploy/tug.sh -COPY scripts /scripts +COPY tug /oc-build-deploy/tug +COPY scripts /oc-build-deploy/scripts -COPY openshift-templates /openshift-templates +COPY openshift-templates /oc-build-deploy/openshift-templates -CMD ["build-deploy"] +CMD ["/oc-build-deploy/build-deploy.sh"] diff --git a/images/oc-build-deploy-dind/build-deploy-docker-compose.sh b/images/oc-build-deploy-dind/build-deploy-docker-compose.sh index 3cde1928e3..5fe0d6bdaa 100755 --- a/images/oc-build-deploy-dind/build-deploy-docker-compose.sh +++ b/images/oc-build-deploy-dind/build-deploy-docker-compose.sh @@ -1,9 +1,14 @@ #!/bin/bash -containsValue () { - local e - for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done - return 1 +function outputToYaml() { + set +x + IFS='' + while read data; do + echo "$data" >> /oc-build-deploy/lagoon/${YAML_CONFIG_FILE}.yml; + done; + # Inject YAML document separator + echo "---" >> /oc-build-deploy/lagoon/${YAML_CONFIG_FILE}.yml; + set -x } ############################################## @@ -13,25 +18,32 @@ containsValue () { # Load path of docker-compose that should be used DOCKER_COMPOSE_YAML=($(cat .lagoon.yml | shyaml get-value docker-compose-yaml)) +DEPLOY_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.deploy-type default) + # Load all Services that are defined -SERVICES=($(cat $DOCKER_COMPOSE_YAML | shyaml keys services)) +COMPOSE_SERVICES=($(cat $DOCKER_COMPOSE_YAML | shyaml keys services)) # Figure out which services should we handle SERVICE_TYPES=() IMAGES=() -for SERVICE in "${SERVICES[@]}" +declare -A MAP_DEPLOYMENT_SERVICETYPE_TO_IMAGENAME +declare -A MAP_SERVICE_TYPE_TO_COMPOSE_SERVICE +declare -A MAP_SERVICE_NAME_TO_IMAGENAME +declare -A IMAGES_PULL +declare -A IMAGES_BUILD +for COMPOSE_SERVICE in "${COMPOSE_SERVICES[@]}" do # The name of the service can be overridden, if not we use the actual servicename - SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.name default) + SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.name default) if [ "$SERVICE_NAME" == "default" ]; then - SERVICE_NAME=$SERVICE + SERVICE_NAME=$COMPOSE_SERVICE fi # Load the servicetype. If it's "none" we will not care about this service at all - SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.type custom) + SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.type custom) # Allow the servicetype to be overriden by environment in .lagoon.yml - ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE false) + ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE_NAME false) if [ ! $ENVIRONMENT_SERVICE_TYPE_OVERRIDE == "false" ]; then SERVICE_TYPE=$ENVIRONMENT_SERVICE_TYPE_OVERRIDE fi @@ -40,19 +52,47 @@ do continue fi - # We build all images - IMAGES+=("${SERVICE}") + # For DeploymentConfigs with multiple Services inside (like nginx-php), we allow to define the service type of within the + # deploymentconfig via lagoon.deployment.servicetype. If this is not set we use the Compose Service Name + DEPLOYMENT_SERVICETYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.deployment\\.servicetype default) + if [ "$DEPLOYMENT_SERVICETYPE" == "default" ]; then + DEPLOYMENT_SERVICETYPE=$COMPOSE_SERVICE + fi + + # The ImageName is the same as the Name of the Docker Compose ServiceName + IMAGE_NAME=$COMPOSE_SERVICE + + # Generate List of Images to build + IMAGES+=("${IMAGE_NAME}") + + # Map Deployment ServiceType to the ImageName + MAP_DEPLOYMENT_SERVICETYPE_TO_IMAGENAME["${SERVICE_NAME}:${DEPLOYMENT_SERVICETYPE}"]="${IMAGE_NAME}" + + # Create an array with all Service Names and Types if it does not exist yet + if [[ ! " ${SERVICE_TYPES[@]} " =~ " ${SERVICE_NAME}:${SERVICE_TYPE} " ]]; then + SERVICE_TYPES+=("${SERVICE_NAME}:${SERVICE_TYPE}") + fi + + # ServiceName and Type to Original Service Name Mapping, but only once per Service name and Type, + # as we have original services that appear twice (like in the case of nginx-php) + if [[ ! "${MAP_SERVICE_TYPE_TO_COMPOSE_SERVICE["${SERVICE_NAME}:${SERVICE_TYPE}"]+isset}" ]]; then + MAP_SERVICE_TYPE_TO_COMPOSE_SERVICE["${SERVICE_NAME}:${SERVICE_TYPE}"]="${COMPOSE_SERVICE}" + fi + + # ServiceName to ImageName mapping, but only once as we have original services that appear twice (like in the case of nginx-php) + # these will be handled via MAP_DEPLOYMENT_SERVICETYPE_TO_IMAGENAME + if [[ ! "${MAP_SERVICE_NAME_TO_IMAGENAME["${SERVICE_NAME}"]+isset}" ]]; then + MAP_SERVICE_NAME_TO_IMAGENAME["${SERVICE_NAME}"]="${IMAGE_NAME}" + fi - # Create an array with all Service Types, Names and Original Service Name - SERVICE_TYPES+=("${SERVICE_TYPE}:${SERVICE_NAME}:${SERVICE}") done ############################################## ### BUILD IMAGES ############################################## -# we only need to build images for pullrequests and branches -if [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then +# we only need to build images for pullrequests and branches, but not during a TUG build +if [[ ( "$TYPE" == "pullrequest" || "$TYPE" == "branch" ) && ! $THIS_IS_TUG == "true" ]]; then BUILD_ARGS=() BUILD_ARGS+=(--build-arg IMAGE_REPO="${CI_OVERRIDE_IMAGE_REPO}") @@ -93,7 +133,9 @@ if [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then PULL_IMAGE=$(echo "${OVERRIDE_IMAGE}" | envsubst) fi - . /scripts/exec-pull-tag.sh + . /oc-build-deploy/scripts/exec-pull-tag.sh + + IMAGES_PULL["${IMAGE_NAME}"]="${PULL_IMAGE}" else # Dockerfile defined, load the context and build it @@ -103,7 +145,9 @@ if [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then echo "defined Dockerfile $DOCKERFILE for service $IMAGE_NAME not found"; exit 1; fi - . /scripts/exec-build.sh + . /oc-build-deploy/scripts/exec-build.sh + + IMAGES_BUILD["${IMAGE_NAME}"]="${TEMPORARY_IMAGE_NAME}" fi # adding the build image to the list of arguments passed into the next image builds @@ -112,35 +156,44 @@ if [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then fi +# if $DEPLOY_TYPE is tug we just push the images to the defined docker registry and create a clone +# of ourselves and push it into `lagoon-tug` image which is then executed in the destination openshift +# If though this is the actual tug deployment in the destination openshift, we don't run this +if [[ $DEPLOY_TYPE == "tug" && ! $THIS_IS_TUG == "true" ]]; then + + . /oc-build-deploy/tug/tug-build-push.sh + + # exit here, we are done + exit +fi + ############################################## ### CREATE OPENSHIFT SERVICES AND ROUTES ############################################## +YAML_CONFIG_FILE="services-routes" + for SERVICE_TYPES_ENTRY in "${SERVICE_TYPES[@]}" do echo "=== BEGIN route processing for service ${SERVICE_TYPES_ENTRY} ===" echo "=== OPENSHIFT_SERVICES_TEMPLATE=${OPENSHIFT_SERVICES_TEMPLATE} " IFS=':' read -ra SERVICE_TYPES_ENTRY_SPLIT <<< "$SERVICE_TYPES_ENTRY" + SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]} + SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]} - SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[0]} - SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[1]} - SERVICE=${SERVICE_TYPES_ENTRY_SPLIT[2]} - - - SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE false) + SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE_NAME false) if [ ! $SERVICE_TYPE_OVERRIDE == "false" ]; then SERVICE_TYPE=$SERVICE_TYPE_OVERRIDE fi - OPENSHIFT_SERVICES_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/services.yml" - + OPENSHIFT_SERVICES_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/services.yml" if [ -f $OPENSHIFT_SERVICES_TEMPLATE ]; then OPENSHIFT_TEMPLATE=$OPENSHIFT_SERVICES_TEMPLATE - . /scripts/exec-openshift-resources.sh + . /oc-build-deploy/scripts/exec-openshift-resources.sh fi - OPENSHIFT_ROUTES_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/routes.yml" + OPENSHIFT_ROUTES_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/routes.yml" if [ -f $OPENSHIFT_ROUTES_TEMPLATE ]; then # The very first generated route is set as MAIN_GENERATED_ROUTE @@ -149,7 +202,7 @@ do fi OPENSHIFT_TEMPLATE=$OPENSHIFT_ROUTES_TEMPLATE - . /scripts/exec-openshift-resources.sh + . /oc-build-deploy/scripts/exec-openshift-resources.sh fi done @@ -185,7 +238,7 @@ while [ -n "$(cat .lagoon.yml | shyaml keys environments.${BRANCH//./\\.}.routes ROUTE_SERVICE=$ROUTES_SERVICE - . /scripts/exec-openshift-create-route.sh + . /oc-build-deploy/scripts/exec-openshift-create-route.sh let ROUTE_DOMAIN_COUNTER=ROUTE_DOMAIN_COUNTER+1 done @@ -193,6 +246,7 @@ while [ -n "$(cat .lagoon.yml | shyaml keys environments.${BRANCH//./\\.}.routes let ROUTES_SERVICE_COUNTER=ROUTES_SERVICE_COUNTER+1 done +oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f /oc-build-deploy/lagoon/${YAML_CONFIG_FILE}.yml ############################################## ### PROJECT WIDE ENV VARIABLES @@ -215,9 +269,9 @@ fi ROUTES=$(oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get routes -o=go-template --template='{{range $index, $route := .items}}{{if $index}},{{end}}{{if $route.spec.tls.termination}}https://{{else}}http://{{end}}{{$route.spec.host}}{{end}}') # Generate a Config Map with project wide env variables -oc process --insecure-skip-tls-verify \ +oc process --local --insecure-skip-tls-verify \ -n ${OPENSHIFT_PROJECT} \ - -f /openshift-templates/configmap.yml \ + -f /oc-build-deploy/openshift-templates/configmap.yml \ -p NAME="lagoon-env" \ -p SAFE_BRANCH="${SAFE_BRANCH}" \ -p SAFE_PROJECT="${SAFE_PROJECT}" \ @@ -238,91 +292,132 @@ fi ############################################## ### PUSH IMAGES TO OPENSHIFT REGISTRY ############################################## +if [[ $THIS_IS_TUG == "true" ]]; then + # Allow to disable registry auth + if [ ! "${TUG_SKIP_REGISTRY_AUTH}" == "true" ]; then + # This adds the defined credentials to the serviceaccount/default so that the deployments can pull from the remote registry + if oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get secret tug-registry 2> /dev/null; then + oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} delete secret tug-registry + fi -if [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then - for IMAGE_NAME in "${IMAGES[@]}" + oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} secrets new-dockercfg tug-registry --docker-server="${TUG_REGISTRY}" --docker-username="${TUG_REGISTRY_USERNAME}" --docker-password="${TUG_REGISTRY_PASSWORD}" --docker-email="${TUG_REGISTRY_USERNAME}" + oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} secrets add serviceaccount/default secrets/tug-registry --for=pull + fi + + # Import all remote Images into ImageStreams + readarray -t TUG_IMAGES < /oc-build-deploy/tug/images + for TUG_IMAGE in "${TUG_IMAGES[@]}" + do + oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} tag --source=docker "${TUG_REGISTRY}/${TUG_REGISTRY_REPOSITORY}/${TUG_IMAGE_PREFIX}${TUG_IMAGE}:${SAFE_BRANCH}" "${TUG_IMAGE}:latest" + done + +elif [ "$TYPE" == "pullrequest" ] || [ "$TYPE" == "branch" ]; then + for IMAGE_NAME in "${!IMAGES_BUILD[@]}" do # Before the push the temporary name is resolved to the future tag with the registry in the image name - TEMPORARY_IMAGE_NAME="${OPENSHIFT_PROJECT}-${IMAGE_NAME}" - . /scripts/exec-push.sh + TEMPORARY_IMAGE_NAME="${IMAGES_BUILD[${IMAGE_NAME}]}" + . /oc-build-deploy/scripts/exec-push-parallel.sh + done + + parallel --retries 4 < /oc-build-deploy/lagoon/push + + for IMAGE_NAME in "${!IMAGES_PULL[@]}" + do + PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" + . /oc-build-deploy/scripts/exec-openshift-tag-dockerhub.sh done elif [ "$TYPE" == "promote" ]; then for IMAGE_NAME in "${IMAGES[@]}" do - . /scripts/exec-openshift-tag.sh + . /oc-build-deploy/scripts/exec-openshift-tag.sh done fi +# Load all Image Hashes for just pushed images +declare -A IMAGE_HASHES +for IMAGE_NAME in "${IMAGES[@]}" +do + IMAGE_HASHES[${IMAGE_NAME}]=$(oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get istag ${IMAGE_NAME}:latest -o go-template --template='{{.image.dockerImageReference}}') +done + ############################################## ### CREATE PVC, DEPLOYMENTS AND CRONJOBS ############################################## +YAML_CONFIG_FILE="deploymentconfigs-pvcs-cronjobs" + for SERVICE_TYPES_ENTRY in "${SERVICE_TYPES[@]}" do IFS=':' read -ra SERVICE_TYPES_ENTRY_SPLIT <<< "$SERVICE_TYPES_ENTRY" - SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[0]} - SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[1]} - SERVICE=${SERVICE_TYPES_ENTRY_SPLIT[2]} + SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]} + SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]} + COMPOSE_SERVICE=${MAP_SERVICE_TYPE_TO_COMPOSE_SERVICE["${SERVICE_TYPES_ENTRY}"]} # Some Templates need additonal Parameters, like where persistent storage can be found. TEMPLATE_PARAMETERS=() - PERSISTENT_STORAGE_PATH=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.persistent false) + PERSISTENT_STORAGE_PATH=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.persistent false) if [ ! $PERSISTENT_STORAGE_PATH == "false" ]; then TEMPLATE_PARAMETERS+=(-p PERSISTENT_STORAGE_PATH="${PERSISTENT_STORAGE_PATH}") - PERSISTENT_STORAGE_CLASS=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.persistent\\.class false) + PERSISTENT_STORAGE_CLASS=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.persistent\\.class false) if [ ! $PERSISTENT_STORAGE_CLASS == "false" ]; then TEMPLATE_PARAMETERS+=(-p PERSISTENT_STORAGE_CLASS="${PERSISTENT_STORAGE_CLASS}") fi - PERSISTENT_STORAGE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.persistent\\.name false) + PERSISTENT_STORAGE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.persistent\\.name false) if [ ! $PERSISTENT_STORAGE_NAME == "false" ]; then TEMPLATE_PARAMETERS+=(-p PERSISTENT_STORAGE_NAME="${PERSISTENT_STORAGE_NAME}") fi - PERSISTENT_STORAGE_SIZE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.persistent\\.size false) + PERSISTENT_STORAGE_SIZE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.persistent\\.size false) if [ ! $PERSISTENT_STORAGE_SIZE == "false" ]; then TEMPLATE_PARAMETERS+=(-p PERSISTENT_STORAGE_SIZE="${PERSISTENT_STORAGE_SIZE}") fi fi - DEPLOYMENT_STRATEGY=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.deployment\\.strategy false) + DEPLOYMENT_STRATEGY=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.deployment\\.strategy false) if [ ! $DEPLOYMENT_STRATEGY == "false" ]; then TEMPLATE_PARAMETERS+=(-p DEPLOYMENT_STRATEGY="${DEPLOYMENT_STRATEGY}") fi # Generate PVC if service type defines one - OPENSHIFT_SERVICES_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/pvc.yml" + OPENSHIFT_SERVICES_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/pvc.yml" if [ -f $OPENSHIFT_SERVICES_TEMPLATE ]; then OPENSHIFT_TEMPLATE=$OPENSHIFT_SERVICES_TEMPLATE - . /scripts/exec-openshift-create-pvc.sh + . /oc-build-deploy/scripts/exec-openshift-create-pvc.sh fi - OPENSHIFT_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/deployment.yml" - OVERRIDE_TEMPLATE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$SERVICE.labels.lagoon\\.template false) + OVERRIDE_TEMPLATE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.template false) if [ "${OVERRIDE_TEMPLATE}" == "false" ]; then - + OPENSHIFT_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/deployment.yml" if [ -f $OPENSHIFT_TEMPLATE ]; then - . /scripts/exec-openshift-create-deployment.sh + . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh fi else OPENSHIFT_TEMPLATE=$OVERRIDE_TEMPLATE if [ ! -f $OPENSHIFT_TEMPLATE ]; then echo "defined template $OPENSHIFT_TEMPLATE for service $SERVICE_TYPE not found"; exit 1; else - . /scripts/exec-openshift-create-deployment.sh + . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh fi fi + # Generate statefulset if service type defines them + OPENSHIFT_STATEFULSET_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/statefulset.yml" + if [ -f $OPENSHIFT_STATEFULSET_TEMPLATE ]; then + OPENSHIFT_TEMPLATE=$OPENSHIFT_STATEFULSET_TEMPLATE + . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh + fi + # Generate cronjobs if service type defines them - OPENSHIFT_SERVICES_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/cronjobs.yml" + OPENSHIFT_SERVICES_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/cronjobs.yml" if [ -f $OPENSHIFT_SERVICES_TEMPLATE ]; then OPENSHIFT_TEMPLATE=$OPENSHIFT_SERVICES_TEMPLATE - . /scripts/exec-openshift-resources.sh + . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh fi ### CUSTOM CRONJOBS @@ -337,7 +432,7 @@ do CRONJOB_SERVICE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.cronjobs.$CRONJOB_COUNTER.service) # Only implement the cronjob for the services we are currently handling - if [ $CRONJOB_SERVICE == $SERVICE ]; then + if [ $CRONJOB_SERVICE == $SERVICE_NAME ]; then # loading original $TEMPLATE_PARAMETERS as multiple cronjobs use the same values TEMPLATE_PARAMETERS=("${DEPLOYMENT_TEMPLATE_PARAMETERS[@]}") @@ -351,15 +446,15 @@ do TEMPLATE_PARAMETERS+=(-p CRONJOB_COMMAND="${CRONJOB_COMMAND}") # Convert the Cronjob Schedule for additional features and better spread - CRONJOB_SCHEDULE=$(/scripts/convert-crontab.sh "$CRONJOB_SCHEDULE") + CRONJOB_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "$CRONJOB_SCHEDULE") TEMPLATE_PARAMETERS+=(-p CRONJOB_SCHEDULE="${CRONJOB_SCHEDULE}") - OPENSHIFT_TEMPLATE="/openshift-templates/${SERVICE_TYPE}/custom-cronjob.yml" + OPENSHIFT_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/custom-cronjob.yml" if [ ! -f $OPENSHIFT_TEMPLATE ]; then echo "No cronjob Template for service type ${SERVICE_TYPE} found"; exit 1; fi - . /scripts/exec-openshift-resources.sh + . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh fi let CRONJOB_COUNTER=CRONJOB_COUNTER+1 @@ -367,6 +462,11 @@ do done +############################################## +### APPLY RESOURCES +############################################## + +oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f /oc-build-deploy/lagoon/${YAML_CONFIG_FILE}.yml ############################################## ### WAIT FOR POST-ROLLOUT TO BE FINISHED @@ -377,8 +477,8 @@ do IFS=':' read -ra SERVICE_TYPES_ENTRY_SPLIT <<< "$SERVICE_TYPES_ENTRY" - SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[0]} - SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[1]} + SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]} + SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]} SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.${SERVICE_NAME}.labels.lagoon\\.rollout deploymentconfigs) @@ -386,13 +486,13 @@ do if [ $SERVICE_TYPE == "mariadb-galera" ]; then STATEFULSET="${SERVICE_NAME}-galera" - . /scripts/exec-monitor-statefulset.sh + . /oc-build-deploy/scripts/exec-monitor-statefulset.sh SERVICE_NAME="${SERVICE_NAME}-maxscale" - . /scripts/exec-monitor-deploy.sh + . /oc-build-deploy/scripts/exec-monitor-deploy.sh elif [ ! $SERVICE_ROLLOUT_TYPE == "false" ]; then - . /scripts/exec-monitor-deploy.sh + . /oc-build-deploy/scripts/exec-monitor-deploy.sh fi done @@ -412,7 +512,7 @@ do SERVICE_NAME=$(cat .lagoon.yml | shyaml get-value tasks.post-rollout.$COUNTER.$TASK_TYPE.service) CONTAINER=$(cat .lagoon.yml | shyaml get-value tasks.post-rollout.$COUNTER.$TASK_TYPE.container false) SHELL=$(cat .lagoon.yml | shyaml get-value tasks.post-rollout.$COUNTER.$TASK_TYPE.shell sh) - . /scripts/exec-post-rollout-tasks-run.sh + . /oc-build-deploy/scripts/exec-post-rollout-tasks-run.sh ;; *) echo "Task Type ${TASK_TYPE} not implemented"; exit 1; diff --git a/images/oc-build-deploy-dind/build-deploy.sh b/images/oc-build-deploy-dind/build-deploy.sh index 9cd108631b..24a8f1b8db 100755 --- a/images/oc-build-deploy-dind/build-deploy.sh +++ b/images/oc-build-deploy-dind/build-deploy.sh @@ -4,6 +4,7 @@ set -eo pipefail OPENSHIFT_REGISTRY=docker-registry.default.svc:5000 OPENSHIFT_PROJECT=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) +REGISTRY_REPOSITORY=$OPENSHIFT_PROJECT if [ "$CI_USE_OPENSHIFT_REGISTRY" == "true" ]; then CI_OVERRIDE_IMAGE_REPO=${OPENSHIFT_REGISTRY}/lagoon @@ -12,9 +13,9 @@ else fi if [ "$TYPE" == "pullrequest" ]; then - /scripts/git-checkout-pull-merge.sh "$SOURCE_REPOSITORY" "$PR_HEAD_SHA" "$PR_BASE_SHA" + /oc-build-deploy/scripts/git-checkout-pull-merge.sh "$SOURCE_REPOSITORY" "$PR_HEAD_SHA" "$PR_BASE_SHA" else - /scripts/git-checkout-pull.sh "$SOURCE_REPOSITORY" "$GIT_REF" + /oc-build-deploy/scripts/git-checkout-pull.sh "$SOURCE_REPOSITORY" "$GIT_REF" fi LAGOON_GIT_SHA=`git rev-parse HEAD` @@ -47,7 +48,7 @@ do ADDITIONAL_YAML_COMMAND=$(cat .lagoon.yml | shyaml get-value additional-yaml.$ADDITIONAL_YAML.command apply) ADDITIONAL_YAML_IGNORE_ERROR=$(cat .lagoon.yml | shyaml get-value additional-yaml.$ADDITIONAL_YAML.ignore_error false) ADDITIONAL_YAML_IGNORE_ERROR="${ADDITIONAL_YAML_IGNORE_ERROR,,}" # convert to lowercase, as shyaml returns "True" if the yaml is set to "true" - . /scripts/exec-additional-yaml.sh + . /oc-build-deploy/scripts/exec-additional-yaml.sh done -. /build-deploy-docker-compose.sh +. /oc-build-deploy/build-deploy-docker-compose.sh diff --git a/images/oc-build-deploy-dind/openshift-templates/cli-persistent/custom-cronjob.yml b/images/oc-build-deploy-dind/openshift-templates/cli-persistent/custom-cronjob.yml index dcc99d7c45..1ef6cd5ce9 100644 --- a/images/oc-build-deploy-dind/openshift-templates/cli-persistent/custom-cronjob.yml +++ b/images/oc-build-deploy-dind/openshift-templates/cli-persistent/custom-cronjob.yml @@ -49,6 +49,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -57,12 +60,12 @@ objects: spec: schedule: "${CRONJOB_SCHEDULE}" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: metadata: - annotations: - alpha.image.policy.openshift.io/resolve-names: "*" labels: cronjob: ${CRONJOB_NAME} branch: ${SAFE_BRANCH} @@ -79,7 +82,7 @@ objects: secretName: lagoon-sshkey containers: - name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME} - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "${CRONJOB_COMMAND}" diff --git a/images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml index c0777f45ce..f97e1692ef 100644 --- a/images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml @@ -48,6 +48,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -81,7 +84,7 @@ objects: defaultMode: 420 secretName: lagoon-sshkey containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} envFrom: - configMapRef: @@ -104,11 +107,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml index 0f3c79f36c..da645f74ff 100644 --- a/images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -64,7 +67,7 @@ objects: defaultMode: 420 secretName: lagoon-sshkey containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} envFrom: - configMapRef: @@ -85,11 +88,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/custom/custom-cronjob.yml b/images/oc-build-deploy-dind/openshift-templates/custom/custom-cronjob.yml index b7ff0d7b4a..bc2b4c7b2b 100644 --- a/images/oc-build-deploy-dind/openshift-templates/custom/custom-cronjob.yml +++ b/images/oc-build-deploy-dind/openshift-templates/custom/custom-cronjob.yml @@ -43,6 +43,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of custom service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -51,12 +54,12 @@ objects: spec: schedule: "${CRONJOB_SCHEDULE}" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: metadata: - annotations: - alpha.image.policy.openshift.io/resolve-names: "*" labels: cronjob: ${CRONJOB_NAME} branch: ${SAFE_BRANCH} @@ -65,7 +68,7 @@ objects: spec: containers: - name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME} - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "${CRONJOB_COMMAND}" diff --git a/images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml index a07402086f..6648c05ae3 100644 --- a/images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Recreate" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 9200 @@ -91,12 +94,4 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange status: {} \ No newline at end of file diff --git a/images/oc-build-deploy-dind/openshift-templates/kibana/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/kibana/deployment.yml index 0199710762..c40631874d 100644 --- a/images/oc-build-deploy-dind/openshift-templates/kibana/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/kibana/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 5601 @@ -82,12 +85,4 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange status: {} diff --git a/images/oc-build-deploy-dind/openshift-templates/logstash/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/logstash/deployment.yml index ca3e1fd186..8af249ae76 100644 --- a/images/oc-build-deploy-dind/openshift-templates/logstash/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/logstash/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of logstash service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 9600 @@ -82,12 +85,4 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange status: {} \ No newline at end of file diff --git a/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/cronjobs.yml b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/cronjobs.yml index db8e097654..1b3e1c36b8 100644 --- a/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/cronjobs.yml +++ b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/cronjobs.yml @@ -34,6 +34,9 @@ parameters: - name: RANDOM_MINUTE generate: expression from: "[0-5]{1}[0-9]{1}" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -41,6 +44,8 @@ objects: name: cronjob-${SERVICE_NAME}-backup spec: schedule: "${RANDOM_MINUTE} 1 * * *" + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -59,7 +64,7 @@ objects: claimName: ${SERVICE_NAME}-data containers: - name: cronjob-${SERVICE_NAME}-backup - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "/lagoon/mysql-backup.sh ${SERVICE_NAME}" diff --git a/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml index a0aef9f344..dc3970e136 100644 --- a/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml @@ -34,63 +34,10 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Recreate" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: -- apiVersion: apps/v1beta1 - kind: StatefulSet - metadata: - name: ${SERVICE_NAME}-galera - spec: - serviceName: ${SERVICE_NAME}-galera - replicas: 3 - selector: - matchLabels: - service: ${SERVICE_NAME}-galera - template: - metadata: - labels: - branch: ${SAFE_BRANCH} - project: ${SAFE_PROJECT} - service: ${SERVICE_NAME}-galera - annotations: - alpha.image.policy.openshift.io/resolve-names: "*" - spec: - containers: - - name: ${SERVICE_NAME}-galera - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest - imagePullPolicy: Always - ports: - - containerPort: 3306 - name: mysql - - containerPort: 4444 - name: sst - - containerPort: 4567 - name: replication - - containerPort: 4568 - name: ist - readinessProbe: - exec: - command: - - /usr/share/container-scripts/mysql/readiness-probe.sh - initialDelaySeconds: 15 - timeoutSeconds: 5 - volumeMounts: - - name: ${SERVICE_NAME}-galera - mountPath: /var/lib/mysql - subPath: data - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeClaimTemplates: - - metadata: - name: ${SERVICE_NAME}-galera - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 5Gi - apiVersion: v1 kind: DeploymentConfig metadata: @@ -115,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} command: - /lagoon/entrypoints.bash - /usr/local/bin/maxscale-start @@ -146,11 +93,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME}-maxscale - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange \ No newline at end of file diff --git a/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/statefulset.yml b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/statefulset.yml new file mode 100644 index 0000000000..ac2c8deafb --- /dev/null +++ b/images/oc-build-deploy-dind/openshift-templates/mariadb-galera/statefulset.yml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: Template +metadata: + creationTimestamp: null + name: lagoon-openshift-template-mariadb-galera +parameters: + - name: SERVICE_NAME + description: Name of this service + required: true + - name: SAFE_BRANCH + description: Which branch this belongs to, special chars replaced with dashes + required: true + - name: SAFE_PROJECT + description: Which project this belongs to, special chars replaced with dashes + required: true + - name: BRANCH + description: Which branch this belongs to, original value + required: true + - name: PROJECT + description: Which project this belongs to, original value + required: true + - name: LAGOON_GIT_SHA + description: git hash sha of the current deployment + required: true + - name: SERVICE_ROUTER_URL + description: URL of the Router for this service + value: "" + - name: OPENSHIFT_PROJECT + description: Name of the Project that this service is in + required: true + - name: REGISTRY + description: Registry where Images are pushed to + required: true + - name: DEPLOYMENT_STRATEGY + description: Strategy of Deploymentconfig + value: "Recreate" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true +objects: +- apiVersion: apps/v1beta1 + kind: StatefulSet + metadata: + name: ${SERVICE_NAME}-galera + spec: + serviceName: ${SERVICE_NAME}-galera + replicas: 3 + selector: + matchLabels: + service: ${SERVICE_NAME}-galera + template: + metadata: + labels: + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + service: ${SERVICE_NAME}-galera + spec: + containers: + - name: ${SERVICE_NAME}-galera + image: ${SERVICE_IMAGE} + imagePullPolicy: Always + ports: + - containerPort: 3306 + name: mysql + - containerPort: 4444 + name: sst + - containerPort: 4567 + name: replication + - containerPort: 4568 + name: ist + readinessProbe: + exec: + command: + - /usr/share/container-scripts/mysql/readiness-probe.sh + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: ${SERVICE_NAME}-galera + mountPath: /var/lib/mysql + subPath: data + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeClaimTemplates: + - metadata: + name: ${SERVICE_NAME}-galera + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/images/oc-build-deploy-dind/openshift-templates/mariadb/cronjobs.yml b/images/oc-build-deploy-dind/openshift-templates/mariadb/cronjobs.yml index 4a7f9281f2..f660a6fca2 100644 --- a/images/oc-build-deploy-dind/openshift-templates/mariadb/cronjobs.yml +++ b/images/oc-build-deploy-dind/openshift-templates/mariadb/cronjobs.yml @@ -34,6 +34,9 @@ parameters: - name: RANDOM_MINUTE generate: expression from: "[0-5]{1}[0-9]{1}" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -42,6 +45,8 @@ objects: spec: schedule: "${RANDOM_MINUTE} 1 * * *" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -60,7 +65,7 @@ objects: claimName: ${SERVICE_NAME}-data containers: - name: cronjob-${SERVICE_NAME}-backup - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "/lagoon/mysql-backup.sh ${SERVICE_NAME}" diff --git a/images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml index 1188850357..1c8d63d8cb 100644 --- a/images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Recreate" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -63,7 +66,7 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME}-data containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 3306 @@ -91,11 +94,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/mongo/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/mongo/deployment.yml index 10494f7809..18ddf96cd1 100644 --- a/images/oc-build-deploy-dind/openshift-templates/mongo/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/mongo/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Recreate" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -64,7 +67,7 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME} containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 27017 @@ -87,11 +90,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/deployment.yml index cbe4a0e5e4..6ab1b34060 100644 --- a/images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/deployment.yml @@ -43,6 +43,18 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: NGINX_SERVICE_IMAGE + description: Pullable image of nginx service + required: true + - name: PHP_SERVICE_IMAGE + description: Pullable image of php service + required: true + - name: NGINX_SERVICE_NAME + description: Name of nginx service + value: "nginx" + - name: PHP_SERVICE_NAME + description: Name of php service + value: "php" objects: - apiVersion: v1 kind: DeploymentConfig @@ -72,8 +84,8 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME} containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/nginx:latest - name: nginx + - image: ${NGINX_SERVICE_IMAGE} + name: ${NGINX_SERVICE_NAME} ports: - containerPort: 8080 protocol: TCP @@ -107,8 +119,8 @@ objects: requests: cpu: 10m memory: 10Mi - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/php:latest - name: php + - image: ${PHP_SERVICE_IMAGE} + name: ${PHP_SERVICE_NAME} ports: - containerPort: 9000 protocol: TCP @@ -142,19 +154,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - nginx - from: - kind: ImageStreamTag - name: nginx:latest - type: ImageChange - - imageChangeParams: - automatic: true - containerNames: - - php - from: - kind: ImageStreamTag - name: php:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/nginx-php/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/nginx-php/deployment.yml index a02cf46719..2dd28d4ae9 100644 --- a/images/oc-build-deploy-dind/openshift-templates/nginx-php/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/nginx-php/deployment.yml @@ -34,6 +34,18 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: NGINX_SERVICE_IMAGE + description: Pullable image of nginx service + required: true + - name: PHP_SERVICE_IMAGE + description: Pullable image of php service + required: true + - name: NGINX_SERVICE_NAME + description: Name of nginx service + value: "nginx" + - name: PHP_SERVICE_NAME + description: Name of php service + value: "php" objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,8 +71,8 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/nginx:latest - name: nginx + - image: ${NGINX_SERVICE_IMAGE} + name: ${NGINX_SERVICE_NAME} ports: - containerPort: 8080 protocol: TCP @@ -91,8 +103,8 @@ objects: requests: cpu: 10m memory: 10Mi - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/php:latest - name: php + - image: ${PHP_SERVICE_IMAGE} + name: ${PHP_SERVICE_NAME} ports: - containerPort: 9000 protocol: TCP @@ -123,20 +135,4 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - nginx - from: - kind: ImageStreamTag - name: nginx:latest - type: ImageChange - - imageChangeParams: - automatic: true - containerNames: - - php - from: - kind: ImageStreamTag - name: php:latest - type: ImageChange status: {} diff --git a/images/oc-build-deploy-dind/openshift-templates/nginx/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/nginx/deployment.yml index 1c73b0e13f..79a3fd456b 100644 --- a/images/oc-build-deploy-dind/openshift-templates/nginx/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/nginx/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 8080 @@ -94,12 +97,4 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange status: {} diff --git a/images/oc-build-deploy-dind/openshift-templates/node-persistent/custom-cronjob.yml b/images/oc-build-deploy-dind/openshift-templates/node-persistent/custom-cronjob.yml index 863a034e0d..8cad3774ff 100644 --- a/images/oc-build-deploy-dind/openshift-templates/node-persistent/custom-cronjob.yml +++ b/images/oc-build-deploy-dind/openshift-templates/node-persistent/custom-cronjob.yml @@ -49,6 +49,9 @@ parameters: - name: CRONJOB_COMMAND description: Command of this cronjob required: true + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -57,6 +60,8 @@ objects: spec: schedule: "${CRONJOB_SCHEDULE}" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -75,7 +80,7 @@ objects: claimName: ${SERVICE_NAME} containers: - name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME} - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "${CRONJOB_COMMAND}" diff --git a/images/oc-build-deploy-dind/openshift-templates/node-persistent/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/node-persistent/deployment.yml index 3b1081570e..844e4e8270 100644 --- a/images/oc-build-deploy-dind/openshift-templates/node-persistent/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/node-persistent/deployment.yml @@ -43,6 +43,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -72,7 +75,7 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME} containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 3000 @@ -105,11 +108,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/node/custom-cronjob.yml b/images/oc-build-deploy-dind/openshift-templates/node/custom-cronjob.yml index 2dbba5d044..0554d13b17 100644 --- a/images/oc-build-deploy-dind/openshift-templates/node/custom-cronjob.yml +++ b/images/oc-build-deploy-dind/openshift-templates/node/custom-cronjob.yml @@ -40,6 +40,9 @@ parameters: - name: CRONJOB_COMMAND description: Command of this cronjob required: true + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -48,6 +51,8 @@ objects: spec: schedule: "${CRONJOB_SCHEDULE}" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -62,7 +67,7 @@ objects: spec: containers: - name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME} - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "${CRONJOB_COMMAND}" diff --git a/images/oc-build-deploy-dind/openshift-templates/node/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/node/deployment.yml index 5c7aa8b0d0..7b133c9267 100644 --- a/images/oc-build-deploy-dind/openshift-templates/node/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/node/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 3000 @@ -89,11 +92,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/postgres/cronjobs.yml b/images/oc-build-deploy-dind/openshift-templates/postgres/cronjobs.yml index e79577632f..16475526cb 100644 --- a/images/oc-build-deploy-dind/openshift-templates/postgres/cronjobs.yml +++ b/images/oc-build-deploy-dind/openshift-templates/postgres/cronjobs.yml @@ -34,6 +34,9 @@ parameters: - name: RANDOM_MINUTE generate: expression from: "[0-5]{1}[0-9]{1}" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: batch/v2alpha1 kind: CronJob @@ -42,6 +45,8 @@ objects: spec: schedule: "${RANDOM_MINUTE} 1 * * *" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -60,7 +65,7 @@ objects: claimName: ${SERVICE_NAME} containers: - name: cronjob-${SERVICE_NAME}-backup - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + image: ${SERVICE_IMAGE} command: - /lagoon/cronjob.sh - "/lagoon/postgres-backup.sh ${SERVICE_NAME}" diff --git a/images/oc-build-deploy-dind/openshift-templates/postgres/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/postgres/deployment.yml index 93e47e9267..d8f0759f0e 100644 --- a/images/oc-build-deploy-dind/openshift-templates/postgres/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/postgres/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Recreate" + - name: POSTGRES_SERVICE_IMAGE + description: Pullable image of postgres service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -63,7 +66,7 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME} containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${POSTGRES_SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 5432 @@ -91,11 +94,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/redis/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/redis/deployment.yml index 3255eaed00..f802e1b073 100644 --- a/images/oc-build-deploy-dind/openshift-templates/redis/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/redis/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 6379 @@ -84,11 +87,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml index 44c1a2ed42..b70a8fc606 100644 --- a/images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -63,7 +66,7 @@ objects: persistentVolumeClaim: claimName: ${SERVICE_NAME} containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 8149 @@ -94,11 +97,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/openshift-templates/varnish/deployment.yml b/images/oc-build-deploy-dind/openshift-templates/varnish/deployment.yml index 22366e84d3..7dd98fb8ac 100644 --- a/images/oc-build-deploy-dind/openshift-templates/varnish/deployment.yml +++ b/images/oc-build-deploy-dind/openshift-templates/varnish/deployment.yml @@ -34,6 +34,9 @@ parameters: - name: DEPLOYMENT_STRATEGY description: Strategy of Deploymentconfig value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true objects: - apiVersion: v1 kind: DeploymentConfig @@ -59,7 +62,7 @@ objects: project: ${SAFE_PROJECT} spec: containers: - - image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest + - image: ${SERVICE_IMAGE} name: ${SERVICE_NAME} ports: - containerPort: 8080 @@ -87,11 +90,3 @@ objects: test: false triggers: - type: ConfigChange - - imageChangeParams: - automatic: true - containerNames: - - ${SERVICE_NAME} - from: - kind: ImageStreamTag - name: ${SERVICE_NAME}:latest - type: ImageChange diff --git a/images/oc-build-deploy-dind/scripts/exec-additional-yaml.sh b/images/oc-build-deploy-dind/scripts/exec-additional-yaml.sh index 9978630088..68d23d4e07 100755 --- a/images/oc-build-deploy-dind/scripts/exec-additional-yaml.sh +++ b/images/oc-build-deploy-dind/scripts/exec-additional-yaml.sh @@ -1,6 +1,6 @@ #!/bin/bash -x -oc process --insecure-skip-tls-verify \ +oc process --local -o yaml --insecure-skip-tls-verify \ -n ${OPENSHIFT_PROJECT} \ -f "${ADDITIONAL_YAML_PATH}" \ -p SAFE_BRANCH="${SAFE_BRANCH}" \ diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-create-deployment.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-create-deployment.sh deleted file mode 100755 index 00646097b5..0000000000 --- a/images/oc-build-deploy-dind/scripts/exec-openshift-create-deployment.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -if [ -n "$ROUTER_URL" ]; then - SERVICE_ROUTER_URL=${SERVICE_NAME}.${ROUTER_URL} -else - SERVICE_ROUTER_URL="" -fi - -JSON=$(oc process --insecure-skip-tls-verify \ - -n ${OPENSHIFT_PROJECT} \ - -f ${OPENSHIFT_TEMPLATE} \ - -p SERVICE_NAME="${SERVICE_NAME}" \ - -p SAFE_BRANCH="${SAFE_BRANCH}" \ - -p SAFE_PROJECT="${SAFE_PROJECT}" \ - -p BRANCH="${BRANCH}" \ - -p PROJECT="${PROJECT}" \ - -p LAGOON_GIT_SHA="${LAGOON_GIT_SHA}" \ - -p SERVICE_ROUTER_URL="${SERVICE_ROUTER_URL}" \ - -p REGISTRY="${OPENSHIFT_REGISTRY}" \ - -p OPENSHIFT_PROJECT=${OPENSHIFT_PROJECT} \ - "${TEMPLATE_PARAMETERS[@]}") - -# If the deploymentconfig already exists, remove `image` from all DeploymentConfig Container definition -# As setting this causes OpenShift => 3.7 to think the image has changed even though there is an ImageTrigger -if oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get dc "$SERVICE_NAME" &> /dev/null; then - echo "$JSON" | jq --raw-output 'del(.items[].spec.template.spec.containers[]?.image)' | oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f - -else - echo "$JSON" | oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f - -fi \ No newline at end of file diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-create-pvc.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-create-pvc.sh index 654bd54d16..8037c9ee2c 100644 --- a/images/oc-build-deploy-dind/scripts/exec-openshift-create-pvc.sh +++ b/images/oc-build-deploy-dind/scripts/exec-openshift-create-pvc.sh @@ -2,5 +2,5 @@ # Only generate PVC if it does not exist yet if ! oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get pvc "$SERVICE_NAME" &> /dev/null; then - . /scripts/exec-openshift-resources.sh + . /oc-build-deploy/scripts/exec-openshift-resources.sh fi \ No newline at end of file diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-create-route.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-create-route.sh index 01d9859fb9..3ce1024c07 100644 --- a/images/oc-build-deploy-dind/scripts/exec-openshift-create-route.sh +++ b/images/oc-build-deploy-dind/scripts/exec-openshift-create-route.sh @@ -3,9 +3,9 @@ if oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get route "$ROUTE_DOMAIN" &> /dev/null; then oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} patch route "$ROUTE_DOMAIN" -p "{\"metadata\":{\"annotations\":{\"kubernetes.io/tls-acme\":\"${ROUTE_TLS_ACME}\"}},\"spec\":{\"to\":{\"name\":\"${ROUTE_SERVICE}\"},\"tls\":{\"insecureEdgeTerminationPolicy\":\"${ROUTE_INSECURE}\"}}}" else - oc process --insecure-skip-tls-verify \ + oc process --local -o yaml --insecure-skip-tls-verify \ -n ${OPENSHIFT_PROJECT} \ - -f /openshift-templates/route.yml \ + -f /oc-build-deploy/openshift-templates/route.yml \ -p SAFE_BRANCH="${SAFE_BRANCH}" \ -p SAFE_PROJECT="${SAFE_PROJECT}" \ -p BRANCH="${BRANCH}" \ @@ -16,5 +16,5 @@ else -p ROUTE_SERVICE="${ROUTE_SERVICE}" \ -p ROUTE_TLS_ACME="${ROUTE_TLS_ACME}" \ -p ROUTE_INSECURE="${ROUTE_INSECURE}" \ - | oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f - + | outputToYaml fi \ No newline at end of file diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-resources-with-images.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-resources-with-images.sh new file mode 100755 index 0000000000..b2288f5694 --- /dev/null +++ b/images/oc-build-deploy-dind/scripts/exec-openshift-resources-with-images.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +if [ -n "$ROUTER_URL" ]; then + SERVICE_ROUTER_URL=${SERVICE_NAME}.${ROUTER_URL} +else + SERVICE_ROUTER_URL="" +fi + +# Inject Pullable Images into Template +TEMPLATE_IMAGES_PARAMETERS=() +# First check if we need multiple Images in this Template (check for `_SERVICE_IMAGE` - see underline) +if [[ $(oc process --local -f ${OPENSHIFT_TEMPLATE} --parameters | grep _SERVICE_IMAGE) ]]; then + while read line + do + # SERVICE_IMAGE Parameters are configured like "[SERVICETYPE]_SERVICE_IMAGE", we split the servicetype away and lowercase it + DEPLOYMENT_SERVICETYPE=$(echo $line | awk -F_ '{print $1}' | tr '[:upper:]' '[:lower:]'); + + # Load which pushed image matches this servicetype of this service name + DEPLOYMENT_SERVICETYPE_IMAGE_NAME="${MAP_DEPLOYMENT_SERVICETYPE_TO_IMAGENAME[${SERVICE_NAME}:${DEPLOYMENT_SERVICETYPE}]}" + # Load the Image Hash of the loaded Image + DEPLOYMENT_SERVICETYPE_IMAGE_NAME_HASH="${IMAGE_HASHES[${DEPLOYMENT_SERVICETYPE_IMAGE_NAME}]}" + # Add the Image Hash as Parameter of "[SERVICETYPE]_SERVICE_IMAGE" + TEMPLATE_IMAGES_PARAMETERS+=(-p "${line}=${DEPLOYMENT_SERVICETYPE_IMAGE_NAME_HASH}") + done < <(oc process --local -f ${OPENSHIFT_TEMPLATE} --parameters | grep _SERVICE_IMAGE | awk '{ print $1 }') +# check if we need a single image to inject +elif [[ $(oc process --local -f ${OPENSHIFT_TEMPLATE} --parameters | grep SERVICE_IMAGE) ]]; then + SERVICE_NAME_IMAGE="${MAP_SERVICE_NAME_TO_IMAGENAME[${SERVICE_NAME}]}" + SERVICE_NAME_IMAGE_HASH="${IMAGE_HASHES[${SERVICE_NAME_IMAGE}]}" + TEMPLATE_IMAGES_PARAMETERS+=(-p "SERVICE_IMAGE=${SERVICE_NAME_IMAGE_HASH}") +fi + +oc process --local -o yaml --insecure-skip-tls-verify \ + -n ${OPENSHIFT_PROJECT} \ + -f ${OPENSHIFT_TEMPLATE} \ + -p SERVICE_NAME="${SERVICE_NAME}" \ + -p SAFE_BRANCH="${SAFE_BRANCH}" \ + -p SAFE_PROJECT="${SAFE_PROJECT}" \ + -p BRANCH="${BRANCH}" \ + -p PROJECT="${PROJECT}" \ + -p LAGOON_GIT_SHA="${LAGOON_GIT_SHA}" \ + -p SERVICE_ROUTER_URL="${SERVICE_ROUTER_URL}" \ + -p REGISTRY="${OPENSHIFT_REGISTRY}" \ + -p OPENSHIFT_PROJECT=${OPENSHIFT_PROJECT} \ + "${TEMPLATE_PARAMETERS[@]}" \ + "${TEMPLATE_IMAGES_PARAMETERS[@]}" \ + | outputToYaml diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-resources.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-resources.sh index f956f87602..258fb0bce2 100755 --- a/images/oc-build-deploy-dind/scripts/exec-openshift-resources.sh +++ b/images/oc-build-deploy-dind/scripts/exec-openshift-resources.sh @@ -6,7 +6,7 @@ else SERVICE_ROUTER_URL="" fi -oc process --insecure-skip-tls-verify \ +oc process --local -o yaml --insecure-skip-tls-verify \ -n ${OPENSHIFT_PROJECT} \ -f ${OPENSHIFT_TEMPLATE} \ -p SERVICE_NAME="${SERVICE_NAME}" \ @@ -19,4 +19,4 @@ oc process --insecure-skip-tls-verify \ -p REGISTRY="${OPENSHIFT_REGISTRY}" \ -p OPENSHIFT_PROJECT=${OPENSHIFT_PROJECT} \ "${TEMPLATE_PARAMETERS[@]}" \ - | oc apply --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} -f - + | outputToYaml diff --git a/images/oc-build-deploy-dind/scripts/exec-openshift-tag-dockerhub.sh b/images/oc-build-deploy-dind/scripts/exec-openshift-tag-dockerhub.sh new file mode 100644 index 0000000000..0d5fc5de1b --- /dev/null +++ b/images/oc-build-deploy-dind/scripts/exec-openshift-tag-dockerhub.sh @@ -0,0 +1,2 @@ +#!/bin/bash +oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} tag --source=docker ${PULL_IMAGE} ${OPENSHIFT_PROJECT}/${IMAGE_NAME}:latest diff --git a/images/oc-build-deploy-dind/scripts/exec-push-parallel.sh b/images/oc-build-deploy-dind/scripts/exec-push-parallel.sh new file mode 100755 index 0000000000..eb36eab8e3 --- /dev/null +++ b/images/oc-build-deploy-dind/scripts/exec-push-parallel.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +docker tag ${TEMPORARY_IMAGE_NAME} ${OPENSHIFT_REGISTRY}/${OPENSHIFT_PROJECT}/${IMAGE_NAME}:latest + +echo "docker push ${OPENSHIFT_REGISTRY}/${OPENSHIFT_PROJECT}/${IMAGE_NAME}:latest" >> /oc-build-deploy/lagoon/push + diff --git a/images/oc-build-deploy-dind/scripts/exec-push.sh b/images/oc-build-deploy-dind/scripts/exec-push.sh deleted file mode 100755 index ce8aa24df3..0000000000 --- a/images/oc-build-deploy-dind/scripts/exec-push.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -docker tag ${TEMPORARY_IMAGE_NAME} ${OPENSHIFT_REGISTRY}/${OPENSHIFT_PROJECT}/${IMAGE_NAME}:latest - -for i in {1..4}; do docker push ${OPENSHIFT_REGISTRY}/${OPENSHIFT_PROJECT}/${IMAGE_NAME}:latest && break || sleep 5; done - diff --git a/images/oc-build-deploy-dind/tug.sh b/images/oc-build-deploy-dind/tug.sh new file mode 100755 index 0000000000..18e8bf7fd6 --- /dev/null +++ b/images/oc-build-deploy-dind/tug.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -x +set -eo pipefail + +THIS_IS_TUG=true + +# Import environment variables with keeping overwritten env variables +TMPFILE=$(mktemp -t dotenv.XXXXXXXX) +export -p > $TMPFILE + +# set -a is short for `set -o allexport` which will export all variables in a file +set -a +. /oc-build-deploy/tug/env +set +a + +# now export all previously existing environments variables so they are stronger than maybe existing ones in the dotenv files +. $TMPFILE || true +# remove the tmpfile +rm $TMPFILE + + + +OPENSHIFT_REGISTRY=docker-registry.default.svc:5000 +OPENSHIFT_PROJECT=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) +REGISTRY_REPOSITORY=$OPENSHIFT_PROJECT + +if [ "$CI_USE_OPENSHIFT_REGISTRY" == "true" ]; then + CI_OVERRIDE_IMAGE_REPO=${OPENSHIFT_REGISTRY}/lagoon +else + CI_OVERRIDE_IMAGE_REPO="" +fi + +if [ ! -f .lagoon.yml ]; then + echo "no .lagoon.yml file found"; exit 1; +fi + +DEPLOYER_TOKEN=$(cat /var/run/secrets/lagoon/deployer/token) + +oc login --insecure-skip-tls-verify --token="${DEPLOYER_TOKEN}" https://kubernetes.default.svc + +. /oc-build-deploy/build-deploy-docker-compose.sh diff --git a/images/oc-build-deploy-dind/tug/Dockerfile b/images/oc-build-deploy-dind/tug/Dockerfile new file mode 100644 index 0000000000..25aeab503e --- /dev/null +++ b/images/oc-build-deploy-dind/tug/Dockerfile @@ -0,0 +1,14 @@ +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-amazeeio}/oc + +ENV LAGOON=oc-build-deploy-tug + +RUN mkdir -p /oc-build-deploy + +# Copying already checked out git repo from oc-build-deploy-dind into tug +COPY . /oc-build-deploy + +WORKDIR /oc-build-deploy/git + + +CMD ["/oc-build-deploy/tug.sh"] \ No newline at end of file diff --git a/images/oc-build-deploy-dind/tug/tug-build-push.sh b/images/oc-build-deploy-dind/tug/tug-build-push.sh new file mode 100644 index 0000000000..a61fb33c16 --- /dev/null +++ b/images/oc-build-deploy-dind/tug/tug-build-push.sh @@ -0,0 +1,50 @@ +TUG_REGISTRY=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.tug.registry false) +TUG_REGISTRY_USERNAME=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.tug.username false) +TUG_REGISTRY_PASSWORD=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.tug.password false) +TUG_REGISTRY_REPOSITORY=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.tug.repository false) +TUG_IMAGE_PREFIX=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.tug.image-prefix '') + + +# Login into TUG registry +docker login -u="${TUG_REGISTRY_USERNAME}" -p="${TUG_REGISTRY_PASSWORD}" ${TUG_REGISTRY} +# Overwrite the registry with the tug registry, so Images are pushed to there +OPENSHIFT_REGISTRY=$TUG_REGISTRY +REGISTRY_REPOSITORY=$TUG_REGISTRY_REPOSITORY + +for IMAGE_NAME in "${IMAGES[@]}" +do + # Before the push the temporary name is resolved to the future tag with the registry in the image name + TEMPORARY_IMAGE_NAME="${OPENSHIFT_PROJECT}-${IMAGE_NAME}" + ORIGINAL_IMAGE_NAME="${IMAGE_NAME}" + IMAGE_NAME="${TUG_IMAGE_PREFIX}${IMAGE_NAME}" + IMAGE_TAG="${SAFE_BRANCH}" + . /oc-build-deploy/scripts/exec-push.sh + echo "${ORIGINAL_IMAGE_NAME}" >> /oc-build-deploy/tug/images +done + +# Save the current environment variables so the tug deployment dan us them +echo "TYPE=\"${TYPE}\"" >> /oc-build-deploy/tug/env +echo "SAFE_BRANCH=\"${SAFE_BRANCH}\"" >> /oc-build-deploy/tug/env +echo "BRANCH=\"${BRANCH}\"" >> /oc-build-deploy/tug/env +echo "SAFE_PROJECT=\"${SAFE_PROJECT}\"" >> /oc-build-deploy/tug/env +echo "PROJECT=\"${PROJECT}\"" >> /oc-build-deploy/tug/env +echo "ROUTER_URL=\"${ROUTER_URL}\"" >> /oc-build-deploy/tug/env +echo "ENVIRONMENT_TYPE=\"${ENVIRONMENT_TYPE}\"" >> /oc-build-deploy/tug/env +echo "CI_USE_OPENSHIFT_REGISTRY=\"${CI_USE_OPENSHIFT_REGISTRY}\"" >> /oc-build-deploy/tug/env +echo "LAGOON_GIT_SHA=\"${LAGOON_GIT_SHA}\"" >> /oc-build-deploy/tug/env +echo "TUG_REGISTRY=\"${TUG_REGISTRY}\"" >> /oc-build-deploy/tug/env +echo "TUG_REGISTRY_USERNAME=\"${TUG_REGISTRY_USERNAME}\"" >> /oc-build-deploy/tug/env +echo "TUG_REGISTRY_PASSWORD=\"${TUG_REGISTRY_PASSWORD}\"" >> /oc-build-deploy/tug/env +echo "TUG_REGISTRY_REPOSITORY=\"${TUG_REGISTRY_REPOSITORY}\"" >> /oc-build-deploy/tug/env +echo "TUG_IMAGE_PREFIX=\"${TUG_IMAGE_PREFIX}\"" >> /oc-build-deploy/tug/env + +# build the tug docker image +IMAGE_NAME="${TUG_IMAGE_PREFIX}lagoon-tug" +BUILD_CONTEXT="/oc-build-deploy/" +DOCKERFILE="tug/Dockerfile" +BUILD_ARGS=() +BUILD_ARGS+=(--build-arg IMAGE_REPO="${CI_OVERRIDE_IMAGE_REPO}") +TEMPORARY_IMAGE_NAME="${OPENSHIFT_PROJECT}-${IMAGE_NAME}" +. /oc-build-deploy/scripts/exec-build.sh +IMAGE_TAG="${SAFE_BRANCH}" +. /oc-build-deploy/scripts/exec-push.sh diff --git a/images/oc/Dockerfile b/images/oc/Dockerfile index d0a336ad81..e71860872c 100644 --- a/images/oc/Dockerfile +++ b/images/oc/Dockerfile @@ -31,7 +31,7 @@ ENV OC_VERSION=v3.6.0 \ # To run the openshift client library `oc` we need glibc, install that first. Copied from https://github.com/jeanblanchard/docker-alpine-glibc/blob/master/Dockerfile RUN apk add -U --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing aufs-util && \ - apk add --update curl jq && \ + apk add --update curl jq parallel && \ curl -Lo /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ curl -Lo glibc.apk "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk" && \ curl -Lo glibc-bin.apk "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-bin-${GLIBC_VERSION}.apk" && \ diff --git a/images/php/fpm/check_fcgi b/images/php/fpm/check_fcgi index d94ea7258b..19caef9f9e 100755 --- a/images/php/fpm/check_fcgi +++ b/images/php/fpm/check_fcgi @@ -1,3 +1,8 @@ #!/bin/sh -v + +# cgi-fcgi has issues with very big environment variables. +# LAGOON_ROUTES can get quite long in for lagoon projects with many routes, so we set it to zero (it's not used by cgi-fcgi) +LAGOON_ROUTES= + # This script calls the /ping endpoing of the php-fpm, if the return code is 0, the php-fpm has correctly started SCRIPT_NAME=/${1:-ping} SCRIPT_FILENAME=/${1:-ping} REQUEST_METHOD=GET /usr/bin/cgi-fcgi -bind -connect 127.0.0.1:9000 \ No newline at end of file diff --git a/local-dev/api-data/api-data.gql b/local-dev/api-data/api-data.gql index 19dcd97175..3d1ff2c131 100644 --- a/local-dev/api-data/api-data.gql +++ b/local-dev/api-data/api-data.gql @@ -159,8 +159,23 @@ mutation FillAPI { id } + CiDrupalGalera: addProject(input: {id: 14, name: "ci-drupal-galera", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-galera.git"}) { + id + } + CiDrupalGaleraSlack: addNotificationToProject(input: {project: "ci-drupal-galera", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) { + id + } + + CiDrupalPostgres: addProject(input: {id: 15, name: "ci-drupal-postgres", customer: 3, openshift: 2, git_url: "ssh://git@192.168.99.1:2222/git/drupal-postgres.git"}) { + id + } + CiDrupalPostgresSlack: addNotificationToProject(input: {project: "ci-drupal-postgres", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) { + id + } + #### Lagoon Kickstart Objects - KickstartCustomer: addCustomer(input: {id: 4, name: "kickstart-customer"}) { + # Customer with a private key that has access to the local-git server. + KickstartCustomer: addCustomer(input: {id: 4, name: "kickstart-customer", private_key: "-----BEGIN RSA PRIVATE KEY-----\nMIIJKAIBAAKCAgEAxGZZrOV7Islo5p51Moabfd1YB8qbHvQZfJDZJmSU4jNxMf8G\nQH8KIM6ndi60xIiDlkh9R50Gs0fgnPaBwpjMjcUay5EvzBJdMmd/EPhg359+4f5Z\nQiGTVaB5UoGJKg9DEK4Xxi+QLpQ1CiJXvd3QOqnQlJZ2WYwz4kdLxF0d+sRrl+0G\nAISt9Gg9kq6wa7k7RRr4/OyD/9DhDr1GXvHXFPRv6QqKq084CqiUaarP7OcbZKi5\nEyMkf0s86ZTttQPqQijWsenLAw6t7J0vM38eojPDMFX4fJ7wVvbHmsdcwb2YxwD0\nk7I85mV5uM99v7owJ/0YSJGcN2YESq8c68rbRp1iSfDhchd9SUyYMFlrj3R+dxte\nTwvN2W5hIEld8Ip+dUWUVaaTeLkFLmqmVhvcMJNmuj+Wnp9USwki6U5HdpgBJPT5\nYJia3/LyE5IlPaRfZ+vBDQqKOeeqkncVtqcZ468ylT0qpqjtV4OSuCzl+P/TeJ+K\npCxDoqp88yQpYRYn9ztvEVvOkT8RERnT0/NVCNKAwHFOXrkK/BJs/h3fj2NddeVC\nJXdwiB4m+T2C/RHtGxVColqUf2nEntXxweuJgqBz+4YxXqRrKu4p5L4NuudcFAyg\nbIRpbaTZDI+vmmXnTXNP6ymMo5fNJ0/IPEBAoDkkc6ZmKdM5Yth6RaCzolsCAwEA\nAQKCAgBRL4n0SaxcFevyISCLMU+HeP8RwXPcGMWMU4ggMcXFt8QcCbK46Ir0QtjA\nps/8yf2zGuYGu2dwps63s8KyAV3VYNwRuEOM1S6HTncdOb850YGuw+h7yMtRwxND\nwt5Db6mmdIDJYRH13zgJp2ajytRv25CoS4ZCwA0WhahRVLi214kudUFc53vNI2YI\ng4PUE+7nQx4X12E9V0ghQqabStdBB0ZXjA8Ef6vH5CXthDmNUX9mXcSbn5RPtndI\ni1Kz2Bl3HdCaHO3ZprDItbU0UWEFZeZSzcb5JO5u1HZwiebTA5/q638uqqg4CUyG\n0+bEYZ/Ud/AY13A/CkHN6ZNH+UQotCH3GLyaTQq6OhyXlgMBojn3xs9iMUclFcuy\nkbZ1jAxqci25pxCIeNDHBDKRyxgSkDPna8ZZ4eKGXjIZzsds4+IDkYJLMg0OCtd2\nKm+ZPM2CFU2YCqt11dlr0higGK/9gdpajJMVgEYAmigQ670LdcBc4XIOpPMrR70a\nPjbF9ze/UqtKDWtz8KMIUcvr0CjhHa3XRgMJiM34hIOZU6xf6rjEgkN2Geq24u1b\nhUW8URb+5dcasQ9iIfkNn3R+zK5PzyvQNf6+XrbVjpLqPHXJYrD85EKlXkPqiE6y\n3ehYMrIaZIY6gMuIVcg8AEtsm5EwQY7ETw4YnMQLKfyweLsHEQKCAQEA5vavDyQj\nn6PqfF1Ntr3N/FfwRr/167HB+/3wHT+FwRpCw62ACeBC0iFXmzG2wfQfot60fUGa\nQEJiUDGZFvBM0tiTpzmgGG7QpRbPe1919Sl5LZeLA9bngRnmqn5zAkmVTeBCO/4F\nMSk9hnBZ0v0W3DqNmjuIH298g3gl4VJeKpILd62LbhjvhjT/LXlekYDEj3p9Xob8\n1OQxFJgftFyg4P07RRaUzsNLhqEdY3VxDcUMb9459fEYeb7sYig+zVPaZQ31aMVK\nj6K+XiH5M5uKJWkPdMDUG84jreFAdBY9kGCuSulTWgmTLlMKSI85q5hkckY2EQQL\n5u456xfyuFcnEwKCAQEA2bCCcqcGIAiMwk/6z7XIJeUsSUg+ObvkEdogk5n6Y1Ea\nt5LxMpQzhG6SHJ2f38VyKgv9e/jnwXI8eiejper6OeQEBG4+AedcLYi0V5SUMIgD\nX4bxT9+qCwYrwt9YHkJySk237WZUWJPVfxHg0vqNYyD/CXBowx0nm8jEuZ8iT+CW\nO2uZq+0DO2WqoYT54lZux6aEzm+oAkzwJJVXJcUVPg7bJXK1ObOzvHpkZJxHL8+S\nKufzew/CXALKWHoCkEP/P8b7oxjcjQI3KK0EM2fABNvN28+qscqTqQjfAsNw24Ob\nP8rL8amgd/x7iddIbEpOSoLAH1gVoxJXA0oqkC6YmQKCAQEAiIeoomW1nUgTdCLf\nrrfyzUnaoqgVIpf42RKa319OnQD+GJg2iSAFwBuvovE3XN4H2YqW3Thri7LyVP+M\nxM+WSkl2tzLMXcUcF4staXvbyeoTVQ0xQzrFrT53aa/IIsEGvntkC/y0awQ6937w\nylWMLvF6BYNNi2+nBjx+//xl5/pfRwbS1mltJkOr2ODXM2IQT9STyc44JU0jak4m\n58Kd44IuiD+6BaPSwKn7KnEhPIeQO4z9bFJyKn3fVIL/5Pa9smEXAjEmS1Rj/ldM\n7eHzPvwlA9p9SFaKJt5x8G25ROCyB1x4RlBEreyutofcCoDSV+8DRPnEY2XN3RhS\nBgCW+wKCAQAyHrqaDSyS2YUXA/UaN59CYlZk9PYmssTa+16+vRfyH+1H0NQpsgd+\neTq4ze3ORpZ3adVN4svxNQ0JjvDVtZkxFPd70aoDJDKL5UqoU3QfDGHCb75FhpiO\n+ze+IVAXf3Ly+pvbU9Edp8PjAsnBqaA9743orXHhYmgJLRtmJWZv/6R3P9ncxLAW\nz9yOXaBu5TmSTBuNsBV9mhs8QQoUjyDJ7f+0yolBJMfAIGnW5EhbkK31pPvhHIrC\nRn4yCr1mW9F77KNpNMMq0BTFD7jE4SFLvRPThSem0Z5Xr8wwxbDJSa7H7DgyhryE\ng6Qp42AwVpdZ/mqfjNdGeWWBQ2UzVxcZAoIBAHNXgjD3umcKciYRAbltNJrXx9xk\ndAv8I69oEMCy4hCmvKUjy/UI9NqXFjBb/G6VGgh6NUE9o9o27t1Y5Plm0izyCA1M\nDFruaRfjyJ8qjbEifcqRtcF3rzsBiXIwdmbN6qT4PUipN2elpUAd7J1OIwGIIe3u\nCWNyOTNL+2+oQ/Eg1Y99mg3yrsVyOwhynVE80/X5cy07bXXR5rv1x4NKSVbPhlnt\nL6J5iAoqoDKICzjcgF5x3mj9YFWZrC3aRxRrN5RoEgeVdcXeK56UJqXHjmKN++m3\nc8OPEIBZiD8UJuhSNSOLiBFrGz6toy6rpHavqqknGhVWotXsAs1h8LNkBe8=\n-----END RSA PRIVATE KEY-----"}) { id } KickstartSlack: addNotificationSlack(input: {name: "amazeeio--lagoon-kickstart", webhook: "https://hooks.slack.com/services/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", channel: "lagoon-kickstart"}) { diff --git a/local-dev/git/Dockerfile b/local-dev/git/Dockerfile index 8a0ddda987..216d61007e 100644 --- a/local-dev/git/Dockerfile +++ b/local-dev/git/Dockerfile @@ -20,6 +20,8 @@ RUN mkdir -m 700 /git/.ssh && \ git --bare init /git/multiproject.git && \ git --bare init /git/node.git && \ git --bare init /git/drupal.git && \ + git --bare init /git/drupal-galera.git && \ + git --bare init /git/drupal-postgres.git && \ git --bare init /git/nginx.git && \ git --bare init /git/features.git diff --git a/openshift-setup/docker-host-cronjobs.yaml b/openshift-setup/docker-host-cronjobs.yaml index cd8a89b011..398a12eb94 100644 --- a/openshift-setup/docker-host-cronjobs.yaml +++ b/openshift-setup/docker-host-cronjobs.yaml @@ -18,6 +18,8 @@ objects: spec: schedule: "*/15 * * * *" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -46,6 +48,8 @@ objects: spec: schedule: "22 1 * * *" concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 jobTemplate: spec: template: @@ -60,10 +64,10 @@ objects: imagePullPolicy: Always command: - /lagoon/cronjob.sh - - /prune-images.sh" + - /prune-images.sh env: - name: REPOSITORY_TO_UPDATE value: ${REPOSITORY_TO_UPDATE} serviceAccount: cronjob serviceAccountName: cronjob - restartPolicy: OnFailure \ No newline at end of file + restartPolicy: OnFailure diff --git a/openshift-setup/rolebinding.yaml b/openshift-setup/rolebinding.yaml new file mode 100644 index 0000000000..51c5071fc4 --- /dev/null +++ b/openshift-setup/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: RoleBinding +metadata: + name: shared-resource-viewers + namespace: lagoon +groupNames: +- system:authenticated +roleRef: + name: shared-resource-viewer + namespace: lagoon +subjects: +- kind: SystemGroup + name: system:authenticated +userNames: null \ No newline at end of file diff --git a/services/logs2slack/src/readFromRabbitMQ.js b/services/logs2slack/src/readFromRabbitMQ.js index 33f02b56e2..082ecc7575 100644 --- a/services/logs2slack/src/readFromRabbitMQ.js +++ b/services/logs2slack/src/readFromRabbitMQ.js @@ -52,6 +52,9 @@ async function readFromRabbitMQ (msg: RabbitMQMsg, channelWrapperLogs: ChannelWr case "github:push:handled": case "bitbucket:repo:push:handled": case "gitlab:push:handled": + case "gitlab:merge_request:opened:handled": + case "gitlab:merge_request:updated:handled": + case "gitlab:merge_request:closed:handled": case "rest:deploy:receive": case "rest:remove:receive": sendToSlack(project, message, '#E8E8E8', ':information_source:', channelWrapperLogs, msg, appId) diff --git a/services/openshiftremove/src/index.js b/services/openshiftremove/src/index.js index a24cded1ea..b2b691051c 100644 --- a/services/openshiftremove/src/index.js +++ b/services/openshiftremove/src/index.js @@ -105,7 +105,7 @@ const messageConsumer = async function(msg) { for (let deploymentconfig of deploymentconfigs.items) { const deploymentconfigsDelete = Promise.promisify(openshift.ns(openshiftProject).deploymentconfigs(deploymentconfig.metadata.name).delete, { context: openshift.ns(openshiftProject).deploymentconfigs(deploymentconfig.metadata.name) }) - await deploymentconfigsDelete() + await deploymentconfigsDelete({ body: {"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}}) logger.info(`${openshiftProject}: Deleted DeploymentConfig ${deploymentconfig.metadata.name}`); } @@ -113,13 +113,12 @@ const messageConsumer = async function(msg) { const pods = await podsGet() for (let pod of pods.items) { const podDelete = Promise.promisify(kubernetes.ns(openshiftProject).pods(pod.metadata.name).delete, { context: kubernetes.ns(openshiftProject).pods(pod.metadata.name) }) - await podDelete() + await podDelete({ body: {"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}}) logger.info(`${openshiftProject}: Deleted Pod ${pod.metadata.name}`); } const hasZeroPods = () => new Promise(async (resolve, reject) => { const pods = await podsGet() - console.log(pods) if (pods.items.length === 0) { logger.info(`${openshiftProject}: All Pods deleted`); resolve() @@ -136,7 +135,7 @@ const messageConsumer = async function(msg) { } const projectsDelete = Promise.promisify(openshift.projects(openshiftProject).delete, { context: openshift.projects(openshiftProject) }) - await projectsDelete() + await projectsDelete({ body: {"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}}) logger.info(`${openshiftProject}: Project deleted`); sendToLagoonLogs('success', projectName, "", "task:remove-openshift:finished", {}, `*[${projectName}]* remove \`${openshiftProject}\`` diff --git a/tests/files/drupal8-mariadb/docker-compose.yml b/tests/files/drupal8-mariadb/docker-compose.yml index 030a100607..987503814e 100644 --- a/tests/files/drupal8-mariadb/docker-compose.yml +++ b/tests/files/drupal8-mariadb/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./web:/app/web:delegated depends_on: - builder - varnish: + varnish-myname: networks: - amazeeio-network - default @@ -27,7 +27,7 @@ services: lagoon.name: varnish depends_on: - nginx - nginx: + nginx-myname: networks: - amazeeio-network - default @@ -38,6 +38,7 @@ services: lagoon.type: nginx-php-persistent lagoon.persistent: /app/web/sites/default/files/ lagoon.name: nginx + lagoon.deployment.servicetype: nginx volumes: - ./web:/app/web:delegated depends_on: @@ -46,7 +47,7 @@ services: - "8080" environment: - AMAZEEIO_URL=drupal.docker.amazee.io - php: + php-myname: build: context: . dockerfile: Dockerfile.php @@ -54,6 +55,7 @@ services: lagoon.type: nginx-php-persistent lagoon.persistent: /app/web/sites/default/files/ lagoon.name: nginx + lagoon.deployment.servicetype: php volumes: - ./web:/app/web:delegated depends_on: diff --git a/tests/tests/drupal-galera.yaml b/tests/tests/drupal-galera.yaml new file mode 100644 index 0000000000..7233a480cf --- /dev/null +++ b/tests/tests/drupal-galera.yaml @@ -0,0 +1,10 @@ +--- +- include: drupal/drupal.yaml + vars: + testname: "Drupal 8 composer PHP 7.1 - GALERA" + drupal_version: 8 + db: mariadb-galera + php_version: 7.1 + git_repo_name: drupal-galera.git + project: ci-drupal-galera + branch: drupal8-composer-71-galera diff --git a/tests/tests/drupal-postgres.yaml b/tests/tests/drupal-postgres.yaml new file mode 100644 index 0000000000..33d5ac9437 --- /dev/null +++ b/tests/tests/drupal-postgres.yaml @@ -0,0 +1,30 @@ +--- +- include: drupal/drupal.yaml + vars: + testname: "Drupal 8 composer PHP 7.0 - POSTGRES" + drupal_version: 8 + db: postgres + php_version: 7.0 + git_repo_name: drupal-postgres.git + project: ci-drupal-postgres + branch: drupal8-composer-70-postgres + +- include: drupal/drupal.yaml + vars: + testname: "Drupal 8 composer PHP 7.1 - POSTGRES" + drupal_version: 8 + db: postgres + php_version: 7.1 + git_repo_name: drupal-postgres.git + project: ci-drupal-postgres + branch: drupal8-composer-71-postgres + +- include: drupal/drupal.yaml + vars: + testname: "Drupal 8 composer PHP 7.2 - POSTGRES" + drupal_version: 8 + db: postgres + php_version: 7.2 + git_repo_name: drupal-postgres.git + project: ci-drupal-postgres + branch: drupal8-composer-72-postgres diff --git a/tests/tests/drupal.yaml b/tests/tests/drupal.yaml index 2fcf0e1b6b..671b18fafb 100644 --- a/tests/tests/drupal.yaml +++ b/tests/tests/drupal.yaml @@ -29,46 +29,6 @@ project: ci-drupal branch: drupal8-composer-72-mariadb -- include: drupal/drupal.yaml - vars: - testname: "Drupal 8 composer PHP 7.0 - POSTGRES" - drupal_version: 8 - db: postgres - php_version: 7.0 - git_repo_name: drupal.git - project: ci-drupal - branch: drupal8-composer-70-postgres - -- include: drupal/drupal.yaml - vars: - testname: "Drupal 8 composer PHP 7.1 - POSTGRES" - drupal_version: 8 - db: postgres - php_version: 7.1 - git_repo_name: drupal.git - project: ci-drupal - branch: drupal8-composer-71-postgres - -- include: drupal/drupal.yaml - vars: - testname: "Drupal 8 composer PHP 7.1 - GALERA" - drupal_version: 8 - db: mariadb-galera - php_version: 7.1 - git_repo_name: drupal.git - project: ci-drupal - branch: drupal8-composer-71-galera - -- include: drupal/drupal.yaml - vars: - testname: "Drupal 8 composer PHP 7.2 - POSTGRES" - drupal_version: 8 - db: postgres - php_version: 7.2 - git_repo_name: drupal.git - project: ci-drupal - branch: drupal8-composer-72-postgres - - include: drupal/drush.yaml vars: testname: "DRUSH"