Merge remote-tracking branch 'origin/289-elasticsearch-cluster-rebase'

.lagoon.yml

@@ -15,3 +15,8 @@ tasks:
         name: 'check\set elasticsearch watchers'
         command: cd /lagoon/kibana-init/; sh watchers.sh
         service: logs-db-ui
+
+environments:
+  develop:
+    types:
+      logs-db: elasticsearch-cluster

Makefile

@@ -395,7 +395,8 @@ all-tests-list:=	features \
 									gitlab \
 									bitbucket \
 									rest \
-									nginx
+									nginx \
+									elasticsearch
 all-tests = $(foreach image,$(all-tests-list),tests/$(image))
 
 # Run all tests
@@ -414,7 +415,7 @@ tests-list:
 deployment-test-services-main = rabbitmq openshiftremove openshiftbuilddeploy openshiftbuilddeploymonitor logs2slack api ssh auth-server local-git local-api-data-watcher-pusher local-es-kibana-watcher-pusher tests
 
 # All Tests that use REST endpoints
-rest-tests = rest node features nginx
+rest-tests = rest node features nginx elasticsearch
 run-rest-tests = $(foreach image,$(rest-tests),tests/$(image))
 # List of Lagoon Services needed for REST endpoint testing
 deployment-test-services-rest = $(deployment-test-services-main) rest2tasks
@@ -589,6 +590,7 @@ else
 	echo "replacing IP in local-dev/api-data/api-data.gql and docker-compose.yaml with the IP '$$OPENSHIFT_MACHINE_IP'"; \
 	sed -i "s/192.168\.[0-9]\{1,3\}\.[0-9]\{3\}/$${OPENSHIFT_MACHINE_IP}/g" local-dev/api-data/api-data.gql docker-compose.yaml;
 endif
+	./local-dev/minishift/minishift ssh --  '/bin/sh -c "sudo sysctl -w vm.max_map_count=262144"'
 	eval $$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) oc-env); \
 	oc login -u system:admin; \
 	bash -c "echo '{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"name\":\"docker-registry-external\"},\"spec\":{\"ports\":[{\"port\":5000,\"protocol\":\"TCP\",\"targetPort\":5000,\"nodePort\":30000}],\"selector\":{\"docker-registry\":\"default\"},\"sessionAffinity\":\"None\",\"type\":\"NodePort\"}}' | oc --context="default/$$(./local-dev/minishift/minishift --profile $(CI_BUILD_TAG) ip | sed 's/\./-/g'):8443/system:admin" create -n default -f -"; \

images/elasticsearch/Dockerfile

@@ -9,6 +9,9 @@ ENV LAGOON=elasticsearch
 COPY --from=commons /lagoon /lagoon
 COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
 COPY --from=commons /home /home
+COPY peer-finder /bin/
+COPY configure-es.sh /lagoon/
+COPY docker-entrypoint.sh /lagoon/entrypoints/90-elasticsearch.sh
 
 RUN chmod g+w /etc/passwd \
     && mkdir -p /home
@@ -23,3 +26,5 @@ ENV TMPDIR=/tmp TMP=/tmp HOME=/home ENV=/home/.bashrc BASH_ENV=/home/.bashrc
 RUN echo "xpack.security.enabled: false" >> config/elasticsearch.yml
 
 ENV ES_JAVA_OPTS "-Xms200m -Xmx200m"
+
+ENTRYPOINT ["/lagoon/entrypoints.bash", "/usr/local/bin/docker-entrypoint.sh"]

images/elasticsearch/configure-es.sh

@@ -0,0 +1,44 @@
+#! /bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CFG=/usr/share/elasticsearch/config/elasticsearch.yml
+
+
+function join {
+    local IFS="$1"; shift; echo "$*";
+}
+
+HOSTNAME=$(hostname)
+# Parse out cluster name, from service name:
+CLUSTER_NAME="$(hostname -f | cut -d'.' -f2)"
+
+while read -ra LINE; do
+    if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then
+        MY_NAME=$LINE
+    fi
+    PEERS=("${PEERS[@]}" $LINE)
+done
+
+if [ "${#PEERS[@]}" = 1 ]; then
+    ES_CLUSTER_ADDRESS="$MY_NAME"
+else
+    ES_CLUSTER_ADDRESS=$(join , "${PEERS[@]}")
+fi
+
+echo "discovery.zen.ping.unicast.hosts: ${ES_CLUSTER_ADDRESS}" >> ${CFG}
+
+# don't need a restart, we're just writing the conf in case there's an
+# unexpected restart on the node.

images/elasticsearch/docker-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ -z "$POD_NAMESPACE" ]; then
+  # Single container runs in docker
+  echo "POD_NAMESPACE not set, spin up single node"
+else
+  # Is running in Kubernetes/OpenShift, so find all other pods
+  # belonging to the namespace
+  echo "Elasticsearch: Finding peers"
+  K8S_SVC_NAME=$(hostname -f | cut -d"." -f2)
+  echo "Using service name: ${K8S_SVC_NAME}"
+  # copy the pristine version to the one that can be edited
+  /usr/bin/peer-finder -on-start="/lagoon/configure-es.sh" -service=${K8S_SVC_NAME}
+fi
+
+exec $@

images/mariadb/docker-entrypoint.bash

@@ -23,7 +23,6 @@ else
   echo "Using service name: ${K8S_SVC_NAME}"
   # copy the pristine version to the one that can be edited
   cp ${CONTAINER_SCRIPTS_DIR}/galera.cnf ${EXTRA_DEFAULTS_FILE}
-
   /usr/bin/peer-finder -on-start="${CONTAINER_SCRIPTS_DIR}/configure-galera.sh" -service=${K8S_SVC_NAME}
 fi
 

images/oc-build-deploy-dind/build-deploy-docker-compose.sh

@@ -500,6 +500,11 @@ do
     SERVICE_NAME="${SERVICE_NAME}-maxscale"
     . /oc-build-deploy/scripts/exec-monitor-deploy.sh
 
+  elif [ $SERVICE_TYPE == "elasticsearch-cluster" ]; then
+
+    STATEFULSET="${SERVICE_NAME}"
+    . /oc-build-deploy/scripts/exec-monitor-statefulset.sh
+
   elif [ ! $SERVICE_ROLLOUT_TYPE == "false" ]; then
     . /oc-build-deploy/scripts/exec-monitor-deploy.sh
   fi

images/oc-build-deploy-dind/openshift-templates/elasticsearch-cluster/services.yml

@@ -0,0 +1,58 @@
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: lagoon-openshift-template-elasticsearch
+parameters:
+  - name: SERVICE_NAME
+    description: Name of this service
+    required: true
+  - name: SAFE_BRANCH
+    description: Which branch this belongs to, special chars replaced with dashes
+    required: true
+  - name: SAFE_PROJECT
+    description: Which project this belongs to, special chars replaced with dashes
+    required: true
+  - name: BRANCH
+    description: Which branch this belongs to, original value
+    required: true
+  - name: PROJECT
+    description: Which project this belongs to, original value
+    required: true
+  - name: LAGOON_GIT_SHA
+    description: git hash sha of the current deployment
+    required: true
+  - name: SERVICE_ROUTER_URL
+    description: URL of the Router for this service
+    value: ""
+  - name: OPENSHIFT_PROJECT
+    description: Name of the Project that this service is in
+    required: true
+  - name: REGISTRY
+    description: Registry where Images are pushed to
+    required: true
+objects:
+- apiVersion: v1
+  kind: Service
+  metadata:
+    annotations:
+      service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+    creationTimestamp: null
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    clusterIP: None
+    ports:
+    - name: 9200-tcp
+      port: 9200
+      protocol: TCP
+    - name: 9300-tcp
+      port: 9300
+      protocol: TCP
+    selector:
+      service: ${SERVICE_NAME}
+  status:
+    loadBalancer: {}

images/oc-build-deploy-dind/openshift-templates/elasticsearch-cluster/statefulset.yml

@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: lagoon-openshift-template-elasticsearch-cluster
+parameters:
+  - name: SERVICE_NAME
+    description: Name of this service
+    required: true
+  - name: SAFE_BRANCH
+    description: Which branch this belongs to, special chars replaced with dashes
+    required: true
+  - name: SAFE_PROJECT
+    description: Which project this belongs to, special chars replaced with dashes
+    required: true
+  - name: BRANCH
+    description: Which branch this belongs to, original value
+    required: true
+  - name: PROJECT
+    description: Which project this belongs to, original value
+    required: true
+  - name: LAGOON_GIT_SHA
+    description: git hash sha of the current deployment
+    required: true
+  - name: SERVICE_ROUTER_URL
+    description: URL of the Router for this service
+    value: ""
+  - name: OPENSHIFT_PROJECT
+    description: Name of the Project that this service is in
+    required: true
+  - name: REGISTRY
+    description: Registry where Images are pushed to
+    required: true
+  - name: SERVICE_IMAGE
+    description: Pullable image of service
+    required: true
+objects:
+- apiVersion: apps/v1beta1
+  kind: StatefulSet
+  metadata:
+    name: ${SERVICE_NAME}
+  spec:
+    serviceName: ${SERVICE_NAME}
+    replicas: 3
+    selector:
+      matchLabels:
+        service: ${SERVICE_NAME}
+    template:
+      metadata:
+        labels:
+          branch: ${SAFE_BRANCH}
+          project: ${SAFE_PROJECT}
+          service: ${SERVICE_NAME}
+      spec:
+        containers:
+        - name: ${SERVICE_NAME}
+          image: ${SERVICE_IMAGE}
+          imagePullPolicy: Always
+          ports:
+          - containerPort: 9200
+            protocol: TCP
+          - containerPort: 9300
+            protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /_cluster/health?local=true
+              port: 9200
+            initialDelaySeconds: 20
+          livenessProbe:
+            httpGet:
+              path: /_cluster/health?local=true
+              port: 9200
+            initialDelaySeconds: 120
+          envFrom:
+          - configMapRef:
+              name: lagoon-env
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+          volumeMounts:
+          - mountPath: /usr/share/elasticsearch/data
+            name: ${SERVICE_NAME}
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+        volumes:
+        - name: ${SERVICE_NAME}
+          emptyDir: {}

images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml

@@ -94,4 +94,4 @@ objects:
     test: false
     triggers:
     - type: ConfigChange
-  status: {}
+  status: {}

local-dev/api-data/api-data.gql

@@ -155,18 +155,25 @@ mutation FillAPI {
   CiFeatures: addProject(input: {id: 12, name: "ci-features", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/features.git", production_environment:"master"}) {
     id
   }
- CiFeaturesSlack: addNotificationToProject(input: {project: "ci-features", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
+  CiFeaturesSlack: addNotificationToProject(input: {project: "ci-features", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
     id
   }
 
-  CiDrupalGalera: addProject(input: {id: 14, name: "ci-drupal-galera", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/drupal-galera.git"}) {
+  CiElasticsearch: addProject(input: {id: 14, name: "ci-elasticsearch", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/elasticsearch.git", production_environment:"master"}) {
+    id
+  }
+  CiElasticsearchSlack: addNotificationToProject(input: {project: "ci-elasticsearch", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
+    id
+  }
+
+  CiDrupalGalera: addProject(input: {id: 15, name: "ci-drupal-galera", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/drupal-galera.git"}) {
     id
   }
   CiDrupalGaleraSlack: addNotificationToProject(input: {project: "ci-drupal-galera", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {
     id
   }
 
-  CiDrupalPostgres: addProject(input: {id: 15, name: "ci-drupal-postgres", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/drupal-postgres.git"}) {
+  CiDrupalPostgres: addProject(input: {id: 16, name: "ci-drupal-postgres", customer: 3, openshift: 2, git_url: "ssh://[email protected]:2222/git/drupal-postgres.git"}) {
     id
   }
   CiDrupalPostgresSlack: addNotificationToProject(input: {project: "ci-drupal-postgres", notificationType: "slack", notificationName: "amazeeio--lagoon-local-ci"}) {

local-dev/git/Dockerfile

@@ -23,8 +23,9 @@ RUN mkdir -m 700 /git/.ssh && \
 	git --bare init /git/drupal-galera.git && \
 	git --bare init /git/drupal-postgres.git && \
 	git --bare init /git/nginx.git && \
-	git --bare init /git/features.git
+	git --bare init /git/features.git && \
+  git --bare init /git/elasticsearch.git
 
 USER root
 
-COPY authorized-keys.sh /etc/my_init.d/authorized-keys.sh
+COPY authorized-keys.sh /etc/my_init.d/authorized-keys.sh

services/logs-db/Dockerfile

@@ -1,4 +1,4 @@
 ARG IMAGE_REPO
 FROM ${IMAGE_REPO:-lagoon}/elasticsearch
 
-COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
+COPY docker-entrypoint.sh /lagoon/entrypoints/95-logs-db.sh

services/logs-db/conf/elasticsearch.yml → services/logs-db/docker-entrypoint.sh

@@ -1,12 +1,8 @@
-cluster.name: "docker-cluster"
-network.host: 0.0.0.0
+#!/usr/bin/env bash
 
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-discovery.zen.minimum_master_nodes: 1
-xpack.license.self_generated.type: trial
-xpack.security.enabled: false
+set -eo pipefail
+
+cat <<EOT >> /usr/share/elasticsearch/config/elasticsearch.yml
 xpack.notification:
   slack:
     account:
@@ -30,3 +26,4 @@ xpack.notification:
           description: "Elasticsearch Watch notification"
           incident_key: "lagoon_logs_key"
           event_type: trigger
+EOT

tests/checks/check-json-content.yaml

@@ -0,0 +1,11 @@
+---
+- name: "{{ testname }} - Check if URL {{url}} contains json {{expected_key}}:{{expected_value}}"
+  uri:
+   url: "{{ url }}"
+   return_content: yes
+   body_format: json
+  register: response
+  until:  response.json is defined and response.json.number_of_nodes is defined and response.json.number_of_nodes == node_count
+  retries: 30
+  delay: 10
+- debug: msg="Success!!!"

tests/files/elasticsearch-cluster/.lagoon.yml

@@ -0,0 +1 @@
+docker-compose-yaml: docker-compose.yml

tests/files/elasticsearch-cluster/Dockerfile.elasticsearch

@@ -0,0 +1,2 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-amazeeio}/elasticsearch

tests/files/elasticsearch-cluster/Dockerfile.nginx

@@ -0,0 +1,4 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-amazeeio}/nginx
+
+COPY app.conf /etc/nginx/conf.d/app.conf

tests/files/elasticsearch-cluster/app.conf

@@ -0,0 +1,10 @@
+server {
+
+    listen 8080 default_server;
+
+    include /etc/nginx/helpers/*.conf;
+
+    location / {
+        proxy_pass http://elasticsearch:9200/;
+    }
+}