From 2308987e687bcbdab2129786f17f46c66b00fced Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Wed, 11 Jul 2018 18:14:07 -0500 Subject: [PATCH 01/15] we use record-modifier instead of record-reformer --- services/logs-forwarder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/logs-forwarder/Dockerfile b/services/logs-forwarder/Dockerfile index d0e9b9daf2..7d8af83cb4 100644 --- a/services/logs-forwarder/Dockerfile +++ b/services/logs-forwarder/Dockerfile @@ -7,7 +7,7 @@ RUN apk add --update --virtual .build-deps \ && gem install \ fluent-plugin-elasticsearch \ fluent-plugin-secure-forward \ - fluent-plugin-record-reformer \ + fluent-plugin-record-modifier \ && gem sources --clear-all \ && apk del .build-deps \ && rm -rf /var/cache/apk/* \ From 966403fb6a126b3479ee9a17913312180c35a7a5 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Wed, 11 Jul 2018 18:57:28 -0500 Subject: [PATCH 02/15] NodePort needs it's own Service definition --- services/logs2logs-db/.lagoon.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/services/logs2logs-db/.lagoon.yml b/services/logs2logs-db/.lagoon.yml index 356169b010..8cb2323e22 100644 --- a/services/logs2logs-db/.lagoon.yml +++ b/services/logs2logs-db/.lagoon.yml @@ -159,13 +159,25 @@ objects: port: 28777 protocol: TCP targetPort: 28777 - - name: beats-input-module - port: 5044 - protocol: TCP - targetPort: 5044 - type: NodePort selector: service: ${SERVICE_NAME} +- apiVersion: v1 + kind: Service + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + name: ${SERVICE_NAME}-beats + spec: + ports: + - name: beats-input-module + port: 5044 + protocol: TCP + targetPort: 5044 + selector: + service: ${SERVICE_NAME} + type: NodePort - apiVersion: v1 kind: Service metadata: From 79acc015debc2a74e5ec348a7233cffd068983d2 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Thu, 12 Jul 2018 09:42:32 -0500 Subject: [PATCH 03/15] adding lagoon-remote inside lagoon with symlinks --- lagoon-remote/.lagoon.yml | 1 + lagoon-remote/README.md | 60 ++++++++ lagoon-remote/docker-compose.yaml | 31 ++++ lagoon-remote/docker-host | 1 + lagoon-remote/logs-collector | 1 + lagoon-remote/logs-forwarder | 1 + .../logs-forwarder-logstash/.lagoon.yml | 120 +++++++++++++++ .../logs-forwarder-logstash/Dockerfile | 17 +++ .../certs/lumberjack.cert | 17 +++ .../logs-forwarder-logstash/logstash.conf | 17 +++ lagoon-remote/logs-forwarder.remote.yml | 140 ++++++++++++++++++ services/logs-forwarder/.lagoon.yml | 2 +- services/logs2logs-db/.lagoon.yml | 10 +- 13 files changed, 416 insertions(+), 2 deletions(-) create mode 100644 lagoon-remote/.lagoon.yml create mode 100644 lagoon-remote/README.md create mode 100644 lagoon-remote/docker-compose.yaml create mode 120000 lagoon-remote/docker-host create mode 120000 lagoon-remote/logs-collector create mode 120000 lagoon-remote/logs-forwarder create mode 100644 lagoon-remote/logs-forwarder-logstash/.lagoon.yml create mode 100644 lagoon-remote/logs-forwarder-logstash/Dockerfile create mode 100644 lagoon-remote/logs-forwarder-logstash/certs/lumberjack.cert create mode 100644 lagoon-remote/logs-forwarder-logstash/logstash.conf create mode 100644 lagoon-remote/logs-forwarder.remote.yml diff --git a/lagoon-remote/.lagoon.yml b/lagoon-remote/.lagoon.yml new file mode 100644 index 0000000000..ee8cdd661e --- /dev/null +++ b/lagoon-remote/.lagoon.yml @@ -0,0 +1 @@ +docker-compose-yaml: docker-compose.yaml diff --git a/lagoon-remote/README.md b/lagoon-remote/README.md new file mode 100644 index 0000000000..ae71dcda8b --- /dev/null +++ b/lagoon-remote/README.md @@ -0,0 +1,60 @@ +# Lagoon Remote + +## design flowchart +https://docs.google.com/drawings/d/1kMCJn3R2sUtiNYraG9mNce-Od8n_6oq-asoR6ISHn_8/edit + +## details + +There are multiple portions to this repo; + +### collector + +The collector is a fluentd instance configured for `secure_forward` on for +both input and output. The `secure_forward` plugin is configured insecurely +between itself and the DaemonSet nodes. Across openshift clusters, +it is configured with a CA Certificate and requires additional manual +configuration. + + + +### logstash + +#### haproxy + + 1. create router-logs service + ~~~~ + oc apply -n lagoon -f supplemental/lagoon-svc-router-logs.yml + ~~~~ + + 1. The openshift haproxy needs to be configured to forward to logstash. + Update `ROUTER_SYSLOG_ADDRESS` to `router-logs.lagoon.svc:5140`. + ~~~~ + oc -n default edit dc/router + ~~~~ + +Also update the template with #xxx + + + +Additionally, `DESTINATION` needs to be set in in the `lagoon-env` +configmap for the deployed project. In production, this will be +https://logs2logs-lagoon-master.ch.amazee.io . +~~~~ +oc -n lagoon-remote-us edit configmap/lagoon-env +~~~~ + +lagoon project + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: router-logs +spec: + externalName: logstash.lagoon-remote-us-master.svc.cluster.local + sessionAffinity: None + type: ExternalName + + +oc -n default patch deploymentconfig/router \ +-p '{"spec":{"template":{"spec":{"containers":{"env": {"name":"blah", "value":"Baz"}}}}}}'' diff --git a/lagoon-remote/docker-compose.yaml b/lagoon-remote/docker-compose.yaml new file mode 100644 index 0000000000..27f6486bdd --- /dev/null +++ b/lagoon-remote/docker-compose.yaml @@ -0,0 +1,31 @@ +version: '2.3' + +x-lagoon-project: + &lagoon-project lagoon-remote + +services: + logs-forwarder-logstash: + build: + context: logs-forwarder-logstash + dockerfile: Dockerfile + labels: + lagoon.type: custom + lagoon.template: logs-forwarder-logstash/.lagoon.yml + logs-forwarder: + build: + context: logs-forwarder + dockerfile: Dockerfile + labels: + lagoon.type: custom + lagoon.template: logs-forwarder.remote.yml # logs-forwarder in a remote openshift needs a special config + logs-collector: + image: openshift/origin-logging-fluentd:v3.6.1 + labels: + lagoon.type: custom + lagoon.rollout: daemonset + lagoon.template: logs-collector/.lagoon.yml + docker-host: + image: amazeeiolagoon/master-docker-host + labels: + lagoon.type: custom + lagoon.template: docker-host/docker-host.yaml diff --git a/lagoon-remote/docker-host b/lagoon-remote/docker-host new file mode 120000 index 0000000000..759a167bb4 --- /dev/null +++ b/lagoon-remote/docker-host @@ -0,0 +1 @@ +../services/docker-host \ No newline at end of file diff --git a/lagoon-remote/logs-collector b/lagoon-remote/logs-collector new file mode 120000 index 0000000000..1ae11ed6b0 --- /dev/null +++ b/lagoon-remote/logs-collector @@ -0,0 +1 @@ +../services/logs-collector/ \ No newline at end of file diff --git a/lagoon-remote/logs-forwarder b/lagoon-remote/logs-forwarder new file mode 120000 index 0000000000..a468efdde9 --- /dev/null +++ b/lagoon-remote/logs-forwarder @@ -0,0 +1 @@ +../services/logs-forwarder \ No newline at end of file diff --git a/lagoon-remote/logs-forwarder-logstash/.lagoon.yml b/lagoon-remote/logs-forwarder-logstash/.lagoon.yml new file mode 100644 index 0000000000..91c20afe5c --- /dev/null +++ b/lagoon-remote/logs-forwarder-logstash/.lagoon.yml @@ -0,0 +1,120 @@ +apiVersion: v1 +kind: Template +metadata: + creationTimestamp: null + name: lagoon-remote-openshift-template-logstash +parameters: + - name: SERVICE_NAME + description: Name of this service + required: true + - name: SAFE_BRANCH + description: Which branch this belongs to, special chars replaced with dashes + required: true + - name: SAFE_PROJECT + description: Which project this belongs to, special chars replaced with dashes + required: true + - name: BRANCH + description: Which branch this belongs to, original value + required: true + - name: PROJECT + description: Which project this belongs to, original value + required: true + - name: LAGOON_GIT_SHA + description: git hash sha of the current deployment + required: true + - name: SERVICE_ROUTER_URL + description: URL of the Router for this service + value: "" + - name: OPENSHIFT_PROJECT + description: Name of the Project that this service is in + required: true + - name: REGISTRY + description: Registry where Images are pushed to + required: true + - name: DEPLOYMENT_STRATEGY + description: Strategy of Deploymentconfig + value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of logstash service + required: true +objects: +- apiVersion: v1 + kind: DeploymentConfig + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + name: ${SERVICE_NAME} + spec: + replicas: 1 + selector: + service: ${SERVICE_NAME} + strategy: + type: ${DEPLOYMENT_STRATEGY} + template: + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + spec: + containers: + - image: ${SERVICE_IMAGE} + name: ${SERVICE_NAME} + ports: + - containerPort: 9600 + protocol: TCP + readinessProbe: + httpGet: + port: 9600 + initialDelaySeconds: 20 + livenessProbe: + httpGet: + port: 9600 + initialDelaySeconds: 120 + envFrom: + - configMapRef: + name: lagoon-env + env: + - name: SERVICE_NAME + value: ${SERVICE_NAME} + resources: + requests: + cpu: 100m + memory: 100Mi + test: false + triggers: + - type: ConfigChange + status: {} +- apiVersion: v1 + kind: Service + metadata: + creationTimestamp: null + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + name: ${SERVICE_NAME} + spec: + ports: + - name: 9600-tcp + port: 9600 + protocol: TCP + targetPort: 9600 + - name: syslog + port: 5140 + protocol: UDP + targetPort: 5140 + selector: + service: ${SERVICE_NAME} + status: + loadBalancer: {} +- apiVersion: v1 + kind: Service + metadata: + name: router-logs + spec: + externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local + sessionAffinity: None + type: ExternalName diff --git a/lagoon-remote/logs-forwarder-logstash/Dockerfile b/lagoon-remote/logs-forwarder-logstash/Dockerfile new file mode 100644 index 0000000000..89123441bd --- /dev/null +++ b/lagoon-remote/logs-forwarder-logstash/Dockerfile @@ -0,0 +1,17 @@ +FROM amazeeio/logstash + +RUN sed -ibak s/^xpack.*//g /usr/share/logstash/config/logstash.yml + +ENV XPACK_MONITORING_ENABLED=false + +ENV LOGS_FORWARDER_LOGSTASH_TARGET_HOST=url.of.logs-forwader-logstash.target \ + LOGS_FORWARDER_LOGSTASH_TARGET_PORT=30703 + +# Remove default shipped pipeline +RUN rm -f pipeline/logstash.conf + +COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf + +COPY certs/ certs/ + + diff --git a/lagoon-remote/logs-forwarder-logstash/certs/lumberjack.cert b/lagoon-remote/logs-forwarder-logstash/certs/lumberjack.cert new file mode 100644 index 0000000000..36187e0e2c --- /dev/null +++ b/lagoon-remote/logs-forwarder-logstash/certs/lumberjack.cert @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQDrhUaboMuRdjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMTgwNjI3MjMzMTQ3WhcNMjgwNjI0MjMzMTQ3WjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD +xQB0cmt3pstQVWkbpyd8AEtsRtt+f4GLMwVdsG37hRXw+xknMGQWKsIKtxpwShfY +hC6YCaS3ZEkkqctyHZgVJDORe9XmSp+IuFP04Ak8qF/ZfHAaseEysaTRHXJP4YeB +jy7q3ehUGy4DGJimuzkFxc1P02Nk4p0I6lx3+WRi+DwK6jtTOAPEMqQHJZqlQj07 +ZnCfY+Cw0xGy+g8JM+N+l2WRD4Dlhqtm7LdRhlKBG2okSec7s5FojjSkBTAS6wfs +tmhBuhvpS72RWIuUHAExwDjCs4/llRGGWCCUqyn6z6stFD6aF7YNsMy3Gy8UtJ0m +iB6zSxWX6flYKevT+rPhAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACzjllluordk +u0+RJklTJzkJRXTstHnp3R7rNVv8GOqO9eTM0N0TeXHjp+LqMYHoA9ehvz+Pk1Z7 +7JlLyK4/OT7aysNVg/QgZCYOHqj6nGWmwKWjRz9r56DV/0vtdiiwKB7GqvhD7ZLn +W3qseIAzvKlzfwuocLKtBYVLD2llKv3iHiB6C2lRLnzRkYvZP2OgVUSoSNkagLiE +h4tTU1eAulpohjzgUFgv4nDvt6Sp+pa4IjY6Av2MssHoL/UN7X00Spgl6pcBVPc3 +JdoikJA2OWI/JgNtFvFsajHOI4+blcauN2C6E8VGWyCKSODSW8zUgq+TQoNJod20 +79ImYvZ2k1M= +-----END CERTIFICATE----- diff --git a/lagoon-remote/logs-forwarder-logstash/logstash.conf b/lagoon-remote/logs-forwarder-logstash/logstash.conf new file mode 100644 index 0000000000..d33c58a0b1 --- /dev/null +++ b/lagoon-remote/logs-forwarder-logstash/logstash.conf @@ -0,0 +1,17 @@ +input { + udp { + port => 5140 + type => syslog + queue_size => 5000 + receive_buffer_bytes => 26214400 + } +} + +output { + lumberjack { + codec => json + hosts => "${LOGS_FORWARDER_LOGSTASH_TARGET_HOST}" + ssl_certificate => "certs/lumberjack.cert" + port => ${LOGS_FORWARDER_LOGSTASH_TARGET_PORT} + } +} diff --git a/lagoon-remote/logs-forwarder.remote.yml b/lagoon-remote/logs-forwarder.remote.yml new file mode 100644 index 0000000000..329f71a71e --- /dev/null +++ b/lagoon-remote/logs-forwarder.remote.yml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: Template +metadata: + creationTimestamp: null + name: lagoon-remote-openshift-template-fluentd +parameters: + - name: SERVICE_NAME + description: Name of this service + required: true + - name: SAFE_BRANCH + description: Which branch this belongs to, special chars replaced with dashes + required: true + - name: SAFE_PROJECT + description: Which project this belongs to, special chars replaced with dashes + required: true + - name: BRANCH + description: Which branch this belongs to, original value + required: true + - name: PROJECT + description: Which project this belongs to, original value + required: true + - name: LAGOON_GIT_SHA + description: git hash sha of the current deployment + required: true + - name: SERVICE_ROUTER_URL + description: URL of the Router for this service + value: "" + - name: OPENSHIFT_PROJECT + description: Name of the Project that this service is in + required: true + - name: REGISTRY + description: Registry where Images are pushed to + required: true + - name: DEPLOYMENT_STRATEGY + description: Strategy of Deploymentconfig + value: "Rolling" + - name: SERVICE_IMAGE + description: Pullable image of service + required: true +objects: +- apiVersion: v1 + kind: DeploymentConfig + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + name: ${SERVICE_NAME} + spec: + replicas: 1 + selector: + service: ${SERVICE_NAME} + strategy: + type: ${DEPLOYMENT_STRATEGY} + template: + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + spec: + containers: + - image: ${SERVICE_IMAGE} + envFrom: + - configMapRef: + name: lagoon-env + name: ${SERVICE_NAME} + ports: + - containerPort: 24284 + protocol: TCP + resources: + requests: + cpu: 10m + memory: 10Mi + volumeMounts: + - mountPath: /fluentd/etc/ + name: config + volumes: + - configMap: + items: + - key: FLUENT_CONF + path: fluent.conf + name: ${SERVICE_NAME}-config + name: config + triggers: + - type: ConfigChange +- apiVersion: v1 + kind: Service + metadata: + labels: + service: ${SERVICE_NAME} + branch: ${SAFE_BRANCH} + project: ${SAFE_PROJECT} + name: ${SERVICE_NAME} + spec: + ports: + - name: secure-forward + port: 24284 + protocol: TCP + targetPort: 24284 + selector: + service: ${SERVICE_NAME} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: ${SERVICE_NAME}-config + data: + FLUENT_CONF: |- + + log_level info + + + @type secure_forward + @label @FORWARD + shared_key "#{ENV['LOGS_FORWARDER_SHARED_KEY']}" + self_hostname "#{ENV['HOSTNAME']}" + secure true + port 24284 + ca_cert_path "/fluentd/ssl/ca_cert.pem" + ca_private_key_path /fluentd/ssl/ca_key.pem + ca_private_key_passphrase "#{ENV['LOGS_FORWARDER_PRIVATE_KEY_PASSPHRASE']}" + + diff --git a/services/logs-forwarder/.lagoon.yml b/services/logs-forwarder/.lagoon.yml index 27f32cc476..527cfb8b0d 100644 --- a/services/logs-forwarder/.lagoon.yml +++ b/services/logs-forwarder/.lagoon.yml @@ -117,7 +117,7 @@ objects: port: 24284 protocol: TCP targetPort: 24284 - type: NodePort + type: NodePort selector: service: ${SERVICE_NAME} - apiVersion: v1 diff --git a/services/logs2logs-db/.lagoon.yml b/services/logs2logs-db/.lagoon.yml index 8cb2323e22..c3f65ed297 100644 --- a/services/logs2logs-db/.lagoon.yml +++ b/services/logs2logs-db/.lagoon.yml @@ -194,4 +194,12 @@ objects: protocol: UDP targetPort: 5140 selector: - service: ${SERVICE_NAME} \ No newline at end of file + service: ${SERVICE_NAME} +- apiVersion: v1 + kind: Service + metadata: + name: router-logs + spec: + externalName: ${SERVICE_NAME}.${OPENSHIFT_PROJECT}.svc.cluster.local + sessionAffinity: None + type: ExternalName From 24c632a5b1c617041ba30b64e40fb5d651fdfee0 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Thu, 12 Jul 2018 18:52:57 -0500 Subject: [PATCH 04/15] give edit to corredt serviceaccount --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ccdc74cf48..06dcc4a3b5 100644 --- a/Makefile +++ b/Makefile @@ -638,7 +638,7 @@ openshift-lagoon-setup: oc -n lagoon adm policy add-scc-to-user privileged -z logs-collector; \ oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \ oc -n lagoon create serviceaccount lagoon-deployer; \ - oc -n lagoon policy add-role-to-user edit -z openshiftbuilddeploy; \ + oc -n lagoon policy add-role-to-user edit -z lagoon-deployer; \ oc -n lagoon create -f openshift-setup/clusterrole-daemonset-admin.yaml; \ oc -n lagoon adm policy add-cluster-role-to-user daemonset-admin -z lagoon-deployer; \ bash -c "oc process -n lagoon -f services/docker-host/docker-host.yaml | oc -n lagoon apply -f -"; \ From f75b9d8d14da60a2ffc42a6ef9c2f01560a66684 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Thu, 12 Jul 2018 18:53:10 -0500 Subject: [PATCH 05/15] env variables need to be wrapped in "" --- lagoon-remote/logs-forwarder-logstash/logstash.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagoon-remote/logs-forwarder-logstash/logstash.conf b/lagoon-remote/logs-forwarder-logstash/logstash.conf index d33c58a0b1..683744ae8f 100644 --- a/lagoon-remote/logs-forwarder-logstash/logstash.conf +++ b/lagoon-remote/logs-forwarder-logstash/logstash.conf @@ -12,6 +12,6 @@ output { codec => json hosts => "${LOGS_FORWARDER_LOGSTASH_TARGET_HOST}" ssl_certificate => "certs/lumberjack.cert" - port => ${LOGS_FORWARDER_LOGSTASH_TARGET_PORT} + port => "${LOGS_FORWARDER_LOGSTASH_TARGET_PORT}" } } From 86743da2d6ebce8e7b4dcc7f1acca4b95dcfbf91 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Thu, 12 Jul 2018 18:53:19 -0500 Subject: [PATCH 06/15] also define default target shared key --- services/logs-forwarder/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/services/logs-forwarder/Dockerfile b/services/logs-forwarder/Dockerfile index 7d8af83cb4..de1005a9b9 100644 --- a/services/logs-forwarder/Dockerfile +++ b/services/logs-forwarder/Dockerfile @@ -21,6 +21,7 @@ RUN find "/fluentd" -exec chmod g+rw {} \; RUN find "/fluentd" -type d -exec chmod g+x {} + ENV LOGS_FORWARDER_SHARED_KEY=secret \ + LOGS_FORWARDER_TARGET_SHARED_KEY=secret \ LOGS_FORWARDER_PRIVATE_KEY_PASSPHRASE=amazing1 ENTRYPOINT ["/bin/entrypoint.sh"] From 08843820910c3913f7a973bb94d769b278639f03 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Fri, 13 Jul 2018 18:54:29 -0500 Subject: [PATCH 07/15] enable multitenancy --- services/logs-db/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/logs-db/Dockerfile b/services/logs-db/Dockerfile index 9cc7379657..00c616ea6f 100644 --- a/services/logs-db/Dockerfile +++ b/services/logs-db/Dockerfile @@ -9,7 +9,8 @@ RUN bin/elasticsearch-plugin install -b com.floragunn:search-guard-6:6.2.4-22.1 && echo "searchguard.audit.enable_rest: true" >> config/elasticsearch.yml \ && chmod +x plugins/search-guard-6/tools/hash.sh \ && sed -i 's/#kibana:/kibana:/' plugins/search-guard-6/sgconfig/sg_config.yml \ - && sed -i 's/#do_not_fail_on_forbidden: false/do_not_fail_on_forbidden: true/' plugins/search-guard-6/sgconfig/sg_config.yml + && sed -i 's/#do_not_fail_on_forbidden: false/do_not_fail_on_forbidden: true/' plugins/search-guard-6/sgconfig/sg_config.yml \ + && sed -i 's/#multitenancy_enabled: true/multitenancy_enabled: true/' plugins/search-guard-6/sgconfig/sg_config.yml ENV LOGSDB_SLACK_WEBHOOK_URL=https://hooks.slack.com/services/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 \ LOGSDB_SLACK_USERNAME=x-pack \ From c58831c4533e354e9510534a6689fb86d7a960c0 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Fri, 13 Jul 2018 18:54:36 -0500 Subject: [PATCH 08/15] disable global tenant --- services/logs-db-ui/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/logs-db-ui/Dockerfile b/services/logs-db-ui/Dockerfile index 7942df7714..5bdfdc00e2 100644 --- a/services/logs-db-ui/Dockerfile +++ b/services/logs-db-ui/Dockerfile @@ -15,8 +15,11 @@ elasticsearch.password: "${LOGSDB_KIBANASERVER_PASSWORD}"\n\ # Disable SSL verification because we use self-signed demo certificates\n\ elasticsearch.ssl.verificationMode: none\n\ \n\ -# Whitelist the Search Guard Multi Tenancy Header\n\\ -elasticsearch.requestHeadersWhitelist: [ "Authorization", "sgtenant" ]' >> config/kibana.yml +# Whitelist the Search Guard Multi Tenancy Header\n\ +elasticsearch.requestHeadersWhitelist: [ "Authorization", "sgtenant" ]\n\ +\n\ +# Disable searchguard global tenant\n\ +searchguard.multitenancy.tenants.enable_global: false' >> config/kibana.yml RUN bin/kibana-plugin install https://search.maven.org/remotecontent?filepath=com/floragunn/search-guard-kibana-plugin/6.2.4-13/search-guard-kibana-plugin-6.2.4-13.zip From b6070a8dd505a8e5d521adf6669c642fe69841b4 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Fri, 13 Jul 2018 18:57:52 -0500 Subject: [PATCH 09/15] better flusing with 8 threads --- services/logs-forwarder/.lagoon.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/logs-forwarder/.lagoon.yml b/services/logs-forwarder/.lagoon.yml index 527cfb8b0d..fc909790b4 100644 --- a/services/logs-forwarder/.lagoon.yml +++ b/services/logs-forwarder/.lagoon.yml @@ -163,9 +163,11 @@ objects: @type memory timekey 3600 + timekey_wait 0s flush_mode interval flush_interval 1s - overflow_action block + chunk_limit_size 32MB + flush_thread_count 8 id_key viaq_msg_id remove_keys viaq_msg_id, viaq_index_name, kubernetes.master_url From cb99864c7ddb7634f4ad9c4281d9a3dc8e09df84 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Fri, 13 Jul 2018 18:58:31 -0500 Subject: [PATCH 10/15] we need the lumberjack plugin --- services/logs2logs-db/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/logs2logs-db/Dockerfile b/services/logs2logs-db/Dockerfile index dc4fed4eb1..bc991f0bfc 100644 --- a/services/logs2logs-db/Dockerfile +++ b/services/logs2logs-db/Dockerfile @@ -1,7 +1,7 @@ ARG IMAGE_REPO FROM ${IMAGE_REPO:-lagoon}/logstash -RUN bin/logstash-plugin install logstash-input-http +RUN bin/logstash-plugin install logstash-input-lumberjack # Remove default shipped pipeline RUN rm -f pipeline/logstash.conf From 389510067c510ca10030499878664bef8bec9ebe Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Fri, 13 Jul 2018 18:58:46 -0500 Subject: [PATCH 11/15] we're using the lumberjack plugin now, the beats didn't really work --- services/logs2logs-db/pipeline/router-logs.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/logs2logs-db/pipeline/router-logs.conf b/services/logs2logs-db/pipeline/router-logs.conf index d84b1f5f7a..2c0f5074a5 100644 --- a/services/logs2logs-db/pipeline/router-logs.conf +++ b/services/logs2logs-db/pipeline/router-logs.conf @@ -5,11 +5,11 @@ input { queue_size => 5000 receive_buffer_bytes => 26214400 } - beats { + lumberjack { port => 5044 - ssl => true ssl_certificate => "certs/lumberjack.cert" ssl_key => "certs/lumberjack.key" + codec => json } } @@ -34,7 +34,7 @@ filter { } output { - stdout { codec => rubydebug } + # stdout { codec => rubydebug } elasticsearch { user => admin password => "${LOGSDB_ADMIN_PASSWORD}" From d660476d7a9de3bf9b1f2a11fe6881e8744e05f2 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Sun, 15 Jul 2018 15:28:56 +1000 Subject: [PATCH 12/15] hotfix: double defined `router-logs` service --- services/logs2logs-db/.lagoon.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/services/logs2logs-db/.lagoon.yml b/services/logs2logs-db/.lagoon.yml index c3f65ed297..870f7dc9da 100644 --- a/services/logs2logs-db/.lagoon.yml +++ b/services/logs2logs-db/.lagoon.yml @@ -178,23 +178,6 @@ objects: selector: service: ${SERVICE_NAME} type: NodePort -- apiVersion: v1 - kind: Service - metadata: - creationTimestamp: null - labels: - service: ${SERVICE_NAME} - branch: ${SAFE_BRANCH} - project: ${SAFE_PROJECT} - name: router-logs - spec: - ports: - - name: udp-input-module - port: 5140 - protocol: UDP - targetPort: 5140 - selector: - service: ${SERVICE_NAME} - apiVersion: v1 kind: Service metadata: From 2fc0288b0fb8c7ee591a666b0c684e8cb86c4c8f Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Sun, 15 Jul 2018 17:15:40 +1000 Subject: [PATCH 13/15] also need to enable multitenancy in kibana --- services/logs-db-ui/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/logs-db-ui/Dockerfile b/services/logs-db-ui/Dockerfile index 5bdfdc00e2..89d27956d1 100644 --- a/services/logs-db-ui/Dockerfile +++ b/services/logs-db-ui/Dockerfile @@ -18,6 +18,8 @@ elasticsearch.ssl.verificationMode: none\n\ # Whitelist the Search Guard Multi Tenancy Header\n\ elasticsearch.requestHeadersWhitelist: [ "Authorization", "sgtenant" ]\n\ \n\ +# Enable multitenancy\n\ +searchguard.multitenancy.enabled: true\n\ # Disable searchguard global tenant\n\ searchguard.multitenancy.tenants.enable_global: false' >> config/kibana.yml From 8574d93f6cca2725034a096bb513cb6b9169bed5 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Sun, 15 Jul 2018 17:15:59 +1000 Subject: [PATCH 14/15] need to separate user and pass via a comma --- services/logs-db-ui/init/index-patterns.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/logs-db-ui/init/index-patterns.sh b/services/logs-db-ui/init/index-patterns.sh index 5de0fd18b8..9e5761e83c 100755 --- a/services/logs-db-ui/init/index-patterns.sh +++ b/services/logs-db-ui/init/index-patterns.sh @@ -1,21 +1,21 @@ #!/usr/bin/env bash # test for lagoon-logs-* index pattern, create and set to default if it does not exist -until sleep 15; curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "lagoon-logs"; +until sleep 15; curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "lagoon-logs"; do - LAGOON_LOG_ID=$(curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"lagoon-logs-*","timeFieldName":"@timestamp"}}' --compressed \ + LAGOON_LOG_ID=$(curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"lagoon-logs-*","timeFieldName":"@timestamp"}}' --compressed \ | grep -oE '"id":(\d*?,|.*?[^\\]",)' | awk -F'"' '{print $4}') && \ - curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/kibana/settings/defaultIndex' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' -H 'Connection: keep-alive' -H 'DNT: 1' --data-binary "{\"value\":\"$LAGOON_LOG_ID\"}" --compressed + curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/kibana/settings/defaultIndex' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' -H 'Connection: keep-alive' -H 'DNT: 1' --data-binary "{\"value\":\"$LAGOON_LOG_ID\"}" --compressed done # test for service-logs-* index pattern, create if it does not exist -until curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "service-logs"; +until curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "service-logs"; do - curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"service-logs-*","timeFieldName":"@timestamp"}}' --compressed + curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"service-logs-*","timeFieldName":"@timestamp"}}' --compressed done # test for router-logs-* index pattern, create if it does not exist -until curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "router-logs"; +until curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "router-logs"; do - curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"router-logs-*","timeFieldName":"@timestamp"}}' --compressed + curl -u "kibanaserver:$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"router-logs-*","timeFieldName":"@timestamp"}}' --compressed done From 999e07cf3fbd89c1182df7c2c751e47edf6dc50c Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Sun, 15 Jul 2018 17:16:11 +1000 Subject: [PATCH 15/15] it's now called lumberjack --- services/logs2logs-db/.lagoon.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/logs2logs-db/.lagoon.yml b/services/logs2logs-db/.lagoon.yml index 870f7dc9da..1fe970d620 100644 --- a/services/logs2logs-db/.lagoon.yml +++ b/services/logs2logs-db/.lagoon.yml @@ -168,10 +168,10 @@ objects: service: ${SERVICE_NAME} branch: ${SAFE_BRANCH} project: ${SAFE_PROJECT} - name: ${SERVICE_NAME}-beats + name: ${SERVICE_NAME}-lumberjack spec: ports: - - name: beats-input-module + - name: lumberjack-input-module port: 5044 protocol: TCP targetPort: 5044