Merge branch 'master' into docs

Makefile

@@ -638,7 +638,7 @@ openshift-lagoon-setup:
 	oc -n lagoon adm policy add-scc-to-user privileged -z logs-collector; \
 	oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \
 	oc -n lagoon create serviceaccount lagoon-deployer; \
-	oc -n lagoon policy add-role-to-user edit -z openshiftbuilddeploy; \
+	oc -n lagoon policy add-role-to-user edit -z lagoon-deployer; \
 	oc -n lagoon create -f openshift-setup/clusterrole-daemonset-admin.yaml; \
 	oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \
 	bash -c "oc process -n lagoon -f services/docker-host/docker-host.yaml | oc -n lagoon apply -f -"; \

lagoon-remote/.lagoon.yml

@@ -0,0 +1 @@
+docker-compose-yaml: docker-compose.yaml

lagoon-remote/README.md

@@ -0,0 +1,60 @@
+# Lagoon Remote
+
+## design flowchart
+https://docs.google.com/drawings/d/1kMCJn3R2sUtiNYraG9mNce-Od8n_6oq-asoR6ISHn_8/edit
+
+## details
+
+There are multiple portions to this repo;
+
+### collector
+
+The collector is a fluentd instance configured for `secure_forward` on for
+both input and output. The `secure_forward` plugin is configured insecurely
+between itself and the DaemonSet nodes. Across openshift clusters,
+it is configured with a CA Certificate and requires additional manual
+configuration.
+
+
+
+### logstash
+
+#### haproxy
+
+  1. create router-logs service
+  ~~~~
+  oc apply -n lagoon -f supplemental/lagoon-svc-router-logs.yml  
+  ~~~~
+
+  1. The openshift haproxy needs to be configured to forward to logstash.
+  Update `ROUTER_SYSLOG_ADDRESS` to `router-logs.lagoon.svc:5140`.
+  ~~~~
+  oc -n default edit dc/router
+  ~~~~
+
+Also update the template with #xxx
+
+
+
+Additionally, `DESTINATION` needs to be set in in the `lagoon-env`
+configmap for the deployed project.  In production, this will be
+https://logs2logs-lagoon-master.ch.amazee.io .
+~~~~
+oc -n lagoon-remote-us edit configmap/lagoon-env
+~~~~
+
+lagoon project
+
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  name: router-logs
+spec:
+  externalName: logstash.lagoon-remote-us-master.svc.cluster.local
+  sessionAffinity: None
+  type: ExternalName
+
+
+oc -n default patch deploymentconfig/router \
+-p  '{"spec":{"template":{"spec":{"containers":{"env": {"name":"blah", "value":"Baz"}}}}}}''

lagoon-remote/docker-compose.yaml

@@ -0,0 +1,31 @@
+version: '2.3'
+
+x-lagoon-project:
+  &lagoon-project lagoon-remote
+
+services:
+  logs-forwarder-logstash:
+    build:
+      context: logs-forwarder-logstash
+      dockerfile: Dockerfile
+    labels:
+      lagoon.type: custom
+      lagoon.template: logs-forwarder-logstash/.lagoon.yml
+  logs-forwarder:
+    build:
+      context: logs-forwarder
+      dockerfile: Dockerfile
+    labels:
+      lagoon.type: custom
+      lagoon.template: logs-forwarder.remote.yml # logs-forwarder in a remote openshift needs a special config
+  logs-collector:
+    image: openshift/origin-logging-fluentd:v3.6.1
+    labels:
+      lagoon.type: custom
+      lagoon.rollout: daemonset
+      lagoon.template: logs-collector/.lagoon.yml
+  docker-host:
+    image: amazeeiolagoon/master-docker-host
+    labels:
+      lagoon.type: custom
+      lagoon.template: docker-host/docker-host.yaml

lagoon-remote/docker-host

@@ -0,0 +1 @@
+../services/docker-host

lagoon-remote/logs-collector

@@ -0,0 +1 @@
+../services/logs-collector/

lagoon-remote/logs-forwarder

@@ -0,0 +1 @@
+../services/logs-forwarder

lagoon-remote/logs-forwarder-logstash/.lagoon.yml

@@ -0,0 +1,120 @@
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: lagoon-remote-openshift-template-logstash
+parameters:
+  - name: SERVICE_NAME
+    description: Name of this service
+    required: true
+  - name: SAFE_BRANCH
+    description: Which branch this belongs to, special chars replaced with dashes
+    required: true
+  - name: SAFE_PROJECT
+    description: Which project this belongs to, special chars replaced with dashes
+    required: true
+  - name: BRANCH
+    description: Which branch this belongs to, original value
+    required: true
+  - name: PROJECT
+    description: Which project this belongs to, original value
+    required: true
+  - name: LAGOON_GIT_SHA
+    description: git hash sha of the current deployment
+    required: true
+  - name: SERVICE_ROUTER_URL
+    description: URL of the Router for this service
+    value: ""
+  - name: OPENSHIFT_PROJECT
+    description: Name of the Project that this service is in
+    required: true
+  - name: REGISTRY
+    description: Registry where Images are pushed to
+    required: true
+  - name: DEPLOYMENT_STRATEGY
+    description: Strategy of Deploymentconfig
+    value: "Rolling"
+  - name: SERVICE_IMAGE
+    description: Pullable image of logstash service
+    required: true
+objects:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    replicas: 1
+    selector:
+      service: ${SERVICE_NAME}
+    strategy:
+      type: ${DEPLOYMENT_STRATEGY}
+    template:
+      metadata:
+        labels:
+          service: ${SERVICE_NAME}
+          branch: ${SAFE_BRANCH}
+          project: ${SAFE_PROJECT}
+      spec:
+        containers:
+        - image: ${SERVICE_IMAGE}
+          name: ${SERVICE_NAME}
+          ports:
+          - containerPort: 9600
+            protocol: TCP
+          readinessProbe:
+            httpGet:
+              port: 9600
+            initialDelaySeconds: 20
+          livenessProbe:
+            httpGet:
+              port: 9600
+            initialDelaySeconds: 120
+          envFrom:
+          - configMapRef:
+              name: lagoon-env
+          env:
+          - name: SERVICE_NAME
+            value: ${SERVICE_NAME}
+          resources:
+            requests:
+              cpu: 100m
+              memory: 100Mi
+    test: false
+    triggers:
+    - type: ConfigChange
+  status: {}
+- apiVersion: v1
+  kind: Service
+  metadata:
+    creationTimestamp: null
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    ports:
+    - name: 9600-tcp
+      port: 9600
+      protocol: TCP
+      targetPort: 9600
+    - name: syslog
+      port: 5140
+      protocol: UDP
+      targetPort: 5140
+    selector:
+      service: ${SERVICE_NAME}
+  status:
+    loadBalancer: {}
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: router-logs
+  spec:
+    externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local
+    sessionAffinity: None
+    type: ExternalName

lagoon-remote/logs-forwarder-logstash/Dockerfile

@@ -0,0 +1,17 @@
+FROM amazeeio/logstash
+
+RUN sed -ibak s/^xpack.*//g /usr/share/logstash/config/logstash.yml
+
+ENV XPACK_MONITORING_ENABLED=false
+
+ENV LOGS_FORWARDER_LOGSTASH_TARGET_HOST=url.of.logs-forwader-logstash.target \
+    LOGS_FORWARDER_LOGSTASH_TARGET_PORT=30703
+
+# Remove default shipped pipeline
+RUN rm -f pipeline/logstash.conf
+
+COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf
+
+COPY certs/ certs/
+
+

lagoon-remote/logs-forwarder-logstash/certs/lumberjack.cert

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICpDCCAYwCCQDrhUaboMuRdjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMTgwNjI3MjMzMTQ3WhcNMjgwNjI0MjMzMTQ3WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD
+xQB0cmt3pstQVWkbpyd8AEtsRtt+f4GLMwVdsG37hRXw+xknMGQWKsIKtxpwShfY
+hC6YCaS3ZEkkqctyHZgVJDORe9XmSp+IuFP04Ak8qF/ZfHAaseEysaTRHXJP4YeB
+jy7q3ehUGy4DGJimuzkFxc1P02Nk4p0I6lx3+WRi+DwK6jtTOAPEMqQHJZqlQj07
+ZnCfY+Cw0xGy+g8JM+N+l2WRD4Dlhqtm7LdRhlKBG2okSec7s5FojjSkBTAS6wfs
+tmhBuhvpS72RWIuUHAExwDjCs4/llRGGWCCUqyn6z6stFD6aF7YNsMy3Gy8UtJ0m
+iB6zSxWX6flYKevT+rPhAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACzjllluordk
+u0+RJklTJzkJRXTstHnp3R7rNVv8GOqO9eTM0N0TeXHjp+LqMYHoA9ehvz+Pk1Z7
+7JlLyK4/OT7aysNVg/QgZCYOHqj6nGWmwKWjRz9r56DV/0vtdiiwKB7GqvhD7ZLn
+W3qseIAzvKlzfwuocLKtBYVLD2llKv3iHiB6C2lRLnzRkYvZP2OgVUSoSNkagLiE
+h4tTU1eAulpohjzgUFgv4nDvt6Sp+pa4IjY6Av2MssHoL/UN7X00Spgl6pcBVPc3
+JdoikJA2OWI/JgNtFvFsajHOI4+blcauN2C6E8VGWyCKSODSW8zUgq+TQoNJod20
+79ImYvZ2k1M=
+-----END CERTIFICATE-----

lagoon-remote/logs-forwarder-logstash/logstash.conf

@@ -0,0 +1,17 @@
+input {
+  udp {
+    port => 5140
+    type => syslog
+    queue_size => 5000
+    receive_buffer_bytes => 26214400
+  }
+}
+
+output {
+  lumberjack {
+    codec => json
+    hosts => "${LOGS_FORWARDER_LOGSTASH_TARGET_HOST}"
+    ssl_certificate => "certs/lumberjack.cert"
+    port => "${LOGS_FORWARDER_LOGSTASH_TARGET_PORT}"
+  }
+}