reorganize index patterns: logs, metrics, traces and alerts

alerts/rules/log_error.yaml

@@ -3,7 +3,7 @@ es_host: jhipster-elasticsearch
 es_port: 9200
 name: log_error
 type: frequency
-index: logstash-*
+index: logs-*
 # link to a kibana dashboard with correct time settings
 use_kibana4_dashboard: "http://localhost:5601/app/kibana#/dashboard/d712f650-e0eb-11e7-9c68-0b9a0f0c183c"
 num_events: 1

docker-compose.yml

@@ -60,6 +60,7 @@ services:
         build: jhipster-zipkin/
         environment:
             - ES_HOSTS=http://jhipster-elasticsearch:9200
-            - ZIPKIN_UI_LOGS_URL=http://localhost:5601/app/kibana#/discover/d0682f20-e0e9-11e7-9c68-0b9a0f0c183c?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-30d,mode:quick,to:now))&_a=(columns:!(X-B3-TraceId,app_name,level,message),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c,key:logger_name,negate:!t,params:(query:metrics,type:phrase),type:phrase,value:metrics),query:(match:(logger_name:(query:metrics,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'logstash-*',key:X-B3-TraceId,negate:!f,params:(query:'trace',type:phrase),type:phrase,value:'trace'),query:(match:(X-B3-TraceId:(query:'{traceId}',type:phrase))))),index:'logstash-*',interval:auto,query:(language:lucene,query:''),sort:!('@timestamp',asc))
+            - ES_INDEX=traces
+            - ZIPKIN_UI_LOGS_URL=http://localhost:5601/app/kibana#/discover/d0682f20-e0e9-11e7-9c68-0b9a0f0c183c?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-30d,mode:quick,to:now))&_a=(columns:!(X-B3-TraceId,app_name,level,message),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c,key:logger_name,negate:!t,params:(query:metrics,type:phrase),type:phrase,value:metrics),query:(match:(logger_name:(query:metrics,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'logs-*',key:X-B3-TraceId,negate:!f,params:(query:'trace',type:phrase),type:phrase,value:'trace'),query:(match:(X-B3-TraceId:(query:'{traceId}',type:phrase))))),index:'logs-*',interval:auto,query:(language:lucene,query:''),sort:!('@timestamp',asc))
         ports:
             - 9411:9411

jhipster-alerter/Dockerfile

@@ -1,17 +1,18 @@
 FROM python:2.7-alpine
 
-RUN apk update && apk upgrade && apk add bash curl tar musl-dev linux-headers g++ libffi-dev libffi openssl-dev
+RUN apk update && apk upgrade && apk add bash curl tar musl-dev linux-headers g++ libffi-dev libffi openssl-dev libmagic
 
 ENV SET_CONTAINER_TIMEZONE=false \
     CONTAINER_TIMEZONE=UTC \
-    ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/v0.1.33.tar.gz \
+    ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/v0.1.36.tar.gz \
     ELASTALERT_HOME=/opt/elastalert \
     RULES_DIRECTORY=/opt/elastalert/rules \
     ES_HOST=jhipster-elasticsearch \
     USE_SSL=False \
     ES_PORT=9200 \
     ES_USERNAME="" \
-    ES_PASSWORD=""
+    ES_PASSWORD="" \
+    ES_INDEX="alerts"
 
 RUN mkdir -p /opt/elastalert
 

jhipster-alerter/config.yaml

@@ -15,7 +15,7 @@ buffer_time:
 # The index on es_host which is used for metadata storage
 # This can be a unmapped index, but it is recommended that you run
 # elastalert-create-index to set a mapping
-writeback_index: elastalert_status
+writeback_index: alerts
 
 # If an alert fails for some reason, ElastAlert will retry
 # sending the alert until this time period has elapsed

jhipster-alerter/start-elastalert.sh

@@ -24,10 +24,10 @@ if ! curl -f $flags ${ES_HOST}:${ES_PORT} >/dev/null 2>&1
 then
 	echo "ES not available at ${ES_HOST}:${ES_PORT}"
 else
-	if ! curl -f $flags ${ES_HOST}:${ES_PORT}/elastalert_status >/dev/null 2>&1
+	if ! curl -f $flags ${ES_HOST}:${ES_PORT}/${ES_INDEX} >/dev/null 2>&1
 	then
 		echo "Creating Elastalert index in ES..."
-	    elastalert-create-index --index elastalert_status --old-index ""
+	    elastalert-create-index --index ${ES_INDEX} --old-index ""
 	else
 	    echo "Elastalert index already exists in ES."
 	fi

jhipster-console/logtrail.json

@@ -2,7 +2,7 @@
   "index_patterns" : [
     {
       "es": {
-        "default_index": "logstash-*",
+        "default_index": "logs-*",
         "allow_url_parameter": false
       },
       "tail_interval_in_seconds": 10,

jhipster-curator/config/action_file.yml

@@ -2,9 +2,8 @@ actions:
   1:
     action: delete_indices
     description: >-
-       Delete indices older than ${UNIT_COUNT:1} ${UNIT:months} (based on index name), for logstash-
-       prefixed indices. Ignore the error if the filter does not result in an
-       actionable list of indices (ignore_empty_list) and exit cleanly.
+       Delete indices older than ${UNIT_COUNT:1} ${UNIT:months} (based on index name), for logs-*
+       prefixed indices.
     options:
       ignore_empty_list: True
       timeout_override:
@@ -13,7 +12,7 @@ actions:
     filters:
     - filtertype: pattern
       kind: prefix
-      value: logstash-
+      value: logs-
       exclude:
     - filtertype: age
       source: name
@@ -25,9 +24,8 @@ actions:
   2:
     action: delete_indices
     description: >-
-       Delete indices older than ${UNIT_COUNT:1} ${UNIT:months} (based on index name), for zipkin-
-       prefixed indices. Ignore the error if the filter does not result in an
-       actionable list of indices (ignore_empty_list) and exit cleanly.
+       Delete indices older than ${UNIT_COUNT:1} ${UNIT:months} (based on index name), for metrics-*
+       prefixed indices.
     options:
       ignore_empty_list: True
       timeout_override:
@@ -36,7 +34,29 @@ actions:
     filters:
     - filtertype: pattern
       kind: prefix
-      value: zipkin:span-
+      value: metrics-
+      exclude:
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: ${UNIT:months}
+      unit_count: ${UNIT_COUNT:1}
+      exclude:
+  3:
+    action: delete_indices
+    description: >-
+       Delete indices older than ${UNIT_COUNT:1} ${UNIT:months} (based on index name), for traces-
+       prefixed indices.
+    options:
+      ignore_empty_list: True
+      timeout_override:
+      continue_if_exception: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: traces-
       exclude:
     - filtertype: age
       source: name

jhipster-import-dashboards/Dockerfile

@@ -1,8 +1,13 @@
 FROM docker.elastic.co/beats/metricbeat:6.4.1
 WORKDIR /usr/share/metricbeat/
 COPY dashboards/ ./
-COPY wait-for-elasticsearch.sh ./
+COPY wait-for-elasticsearch.sh set-logs-index-pattern-as-default.sh ./
 ENV ELASTICSEARCH_URL=http://jhipster-elasticsearch:9200
 ENV KIBANA_URL=http://jhipster-console:5601
 
-CMD ./wait-for-elasticsearch.sh  && ./metricbeat setup --dashboards -E output.elasticsearch.hosts="[$ELASTICSEARCH_URL]" -E setup.kibana.host=$KIBANA_URL -E setup.dashboards.directory=/usr/share/metricbeat/import
+CMD ./wait-for-elasticsearch.sh \
+    && ./metricbeat setup --dashboards \
+        -E output.elasticsearch.hosts="[$ELASTICSEARCH_URL]" \
+        -E setup.kibana.host=$KIBANA_URL \
+        -E setup.dashboards.directory=/usr/share/metricbeat/import \
+    && ./set-logs-index-pattern-as-default.sh

jhipster-import-dashboards/dashboards/import/6/dashboard/alerting-dashboard.json → jhipster-import-dashboards/dashboards/import/6/dashboard/alerts-dashboard.json

@@ -11,7 +11,7 @@
         "description": "",
         "hits": 0,
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"elastalert-status\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+          "searchSourceJSON": "{\"index\":\"alerts\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
         },
         "sort": [
           "@timestamp",
@@ -29,7 +29,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"elastalert-status\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"alerts\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
         },
         "title": "Alert count by rule name",
         "uiStateJSON": "{\"vis\":{\"colors\":{\"log_error\":\"#C15C17\"}}}",
@@ -50,7 +50,7 @@
         "title": "Select Alert",
         "uiStateJSON": "{}",
         "version": 1,
-        "visState": "{\"title\":\"Select Alert\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1533889809924\",\"indexPattern\":\"elastalert-status\",\"fieldName\":\"rule_name\",\"parent\":\"\",\"label\":\"Rule name\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":5,\"order\":\"desc\"}},{\"id\":\"1533889817560\",\"indexPattern\":\"elastalert-status\",\"fieldName\":\"alert_sent\",\"parent\":\"\",\"label\":\"Alert sent\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":false,\"size\":5,\"order\":\"desc\"}}],\"updateFiltersOnChange\":true,\"useTimeFilter\":false,\"pinFilters\":false},\"aggs\":[]}"
+        "visState": "{\"title\":\"Select Alert\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1533889809924\",\"indexPattern\":\"alerts\",\"fieldName\":\"rule_name\",\"parent\":\"\",\"label\":\"Rule name\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":5,\"order\":\"desc\"}},{\"id\":\"1533889817560\",\"indexPattern\":\"alerts\",\"fieldName\":\"alert_sent\",\"parent\":\"\",\"label\":\"Alert sent\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":false,\"size\":5,\"order\":\"desc\"}}],\"updateFiltersOnChange\":true,\"useTimeFilter\":false,\"pinFilters\":false},\"aggs\":[]}"
       },
       "id": "eaa9d4c0-9c77-11e8-bfa6-a7fdcbb4d022",
       "type": "visualization",
@@ -59,15 +59,15 @@
     },
     {
       "attributes": {
-        "description": "",
+        "description": "View alerts",
         "hits": 0,
         "kibanaSavedObjectMeta": {
           "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
         },
         "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
         "panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":10,\"w\":48,\"h\":19,\"i\":\"1\"},\"embeddableConfig\":{},\"id\":\"739fb300-9c76-11e8-bfa6-a7fdcbb4d022\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":22,\"y\":0,\"w\":26,\"h\":10,\"i\":\"2\"},\"embeddableConfig\":{},\"id\":\"0d103f50-9c77-11e8-bfa6-a7fdcbb4d022\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":0,\"y\":0,\"w\":22,\"h\":10,\"i\":\"4\"},\"version\":\"6.4.0\",\"type\":\"visualization\",\"id\":\"eaa9d4c0-9c77-11e8-bfa6-a7fdcbb4d022\",\"embeddableConfig\":{}}]",
         "timeRestore": false,
-        "title": "alerting-dashboard",
+        "title": "alerts-dashboard",
         "version": 1
       },
       "id": "4161cd50-9c77-11e8-bfa6-a7fdcbb4d022",

jhipster-import-dashboards/dashboards/import/6/dashboard/jvm-dashboard.json

@@ -4,7 +4,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"metrics\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
         },
         "title": "Instances",
         "uiStateJSON": "{}",
@@ -19,7 +19,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.total*\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"metrics\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.total*\",\"language\":\"lucene\"}}"
         },
         "title": "JVM Total Memory",
         "uiStateJSON": "{}",
@@ -34,7 +34,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.heap*\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"metrics\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.heap*\",\"language\":\"lucene\"}}"
         },
         "title": "JVM Heap Memory",
         "uiStateJSON": "{}",
@@ -49,7 +49,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.heap*\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"metrics\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.heap*\",\"language\":\"lucene\"}}"
         },
         "title": "JVM Threads",
         "uiStateJSON": "{}",
@@ -79,7 +79,7 @@
       "attributes": {
         "description": "",
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.non-heap*\",\"language\":\"lucene\"}}"
+          "searchSourceJSON": "{\"index\":\"metrics\",\"filter\":[],\"query\":{\"query\":\"metric_name=jvm.memory.non-heap*\",\"language\":\"lucene\"}}"
         },
         "title": "JVM Non-Heap Memory",
         "uiStateJSON": "{}",

jhipster-import-dashboards/dashboards/import/6/dashboard/logs-dashboard.json

@@ -11,7 +11,7 @@
         "description": "",
         "hits": 0,
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"meta\":{\"index\":\"c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"logger_name\",\"value\":\"metrics\",\"params\":{\"query\":\"metrics\",\"type\":\"phrase\"}},\"query\":{\"match\":{\"logger_name\":{\"query\":\"metrics\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
+          "searchSourceJSON": "{\"index\":\"logs-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"meta\":{\"index\":\"c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"logger_name\",\"value\":\"metrics\",\"params\":{\"query\":\"metrics\",\"type\":\"phrase\"}},\"query\":{\"match\":{\"logger_name\":{\"query\":\"metrics\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
         },
         "sort": [
           "@timestamp",
@@ -35,7 +35,7 @@
         "description": "",
         "hits": 0,
         "kibanaSavedObjectMeta": {
-          "searchSourceJSON": "{\"index\":\"logstash-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"meta\":{\"index\":\"c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"logger_name\",\"value\":\"metrics\",\"params\":{\"query\":\"metrics\",\"type\":\"phrase\"}},\"query\":{\"match\":{\"logger_name\":{\"query\":\"metrics\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
+          "searchSourceJSON": "{\"index\":\"logs-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"meta\":{\"index\":\"c7b73f10-e0e4-11e7-9c68-0b9a0f0c183c\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"logger_name\",\"value\":\"metrics\",\"params\":{\"query\":\"metrics\",\"type\":\"phrase\"}},\"query\":{\"match\":{\"logger_name\":{\"query\":\"metrics\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
         },
         "sort": [
           "@timestamp",
@@ -58,7 +58,7 @@
         "title": "Select Application",
         "uiStateJSON": "{}",
         "version": 1,
-        "visState": "{\"title\":\"Select Application\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1513348424847\",\"indexPattern\":\"logstash-*\",\"fieldName\":\"app_name.keyword\",\"label\":\"Application\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":100,\"order\":\"desc\"}},{\"id\":\"1513956661970\",\"indexPattern\":\"logstash-*\",\"fieldName\":\"instance_name.keyword\",\"label\":\"Instance\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":100,\"order\":\"desc\"}},{\"id\":\"1513958886954\",\"indexPattern\":\"logstash-*\",\"fieldName\":\"level.keyword\",\"label\":\"Log level\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":5,\"order\":\"desc\"}}],\"updateFiltersOnChange\":true},\"aggs\":[]}"
+        "visState": "{\"title\":\"Select Application\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1513348424847\",\"indexPattern\":\"logs-*\",\"fieldName\":\"app_name.keyword\",\"label\":\"Application\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":100,\"order\":\"desc\"}},{\"id\":\"1513956661970\",\"indexPattern\":\"logs-*\",\"fieldName\":\"instance_name.keyword\",\"label\":\"Instance\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":100,\"order\":\"desc\"}},{\"id\":\"1513958886954\",\"indexPattern\":\"logs-*\",\"fieldName\":\"level.keyword\",\"label\":\"Log level\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":5,\"order\":\"desc\"}}],\"updateFiltersOnChange\":true},\"aggs\":[]}"
       },
       "id": "08029590-e1a5-11e7-9e74-19a039e44384",
       "type": "visualization",