Add report-cassandra into monitoring (#3019)

* Update additional-scrape-configs.yaml * Update cassandraRules.yaml * Update alertrules.yaml * Update cassandraRules.yaml * Update cassandraRules.yaml * Update cassandraRules.yaml * Update cassandraRules.yaml * Update cassandraRules.yaml * Update alertrules.yaml * Update alertrules.yaml * Update alertrules.yaml * Update common.yml
project-sunbird · Dec 8, 2021 · 5c6cd0e · 5c6cd0e
1 parent 4556a2b
commit 5c6cd0e
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/additional-scrape-configs.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/additional-scrape-configs.yaml
@@ -53,6 +53,13 @@ scrapeconfig:
     metrics_path: /metrics
     static_configs:
       - targets:  ["{{ groups['cassandra'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:8080' ) | list | join("\", \"") }}"]
+
+{% if 'report-cassandra' in groups and groups['report-cassandra'] %}
+  - job_name: 'report-cassandra-exporter'
+    metrics_path: /metrics
+    static_configs:
+      - targets:  ["{{ groups['report-cassandra'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:8080' ) | list | join("\", \"") }}"]
+{% endif %}
 
 {% if 'druid' in groups and groups['druid'] %}
   - job_name: 'druid-exporter'

diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/cassandraRules.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/cassandraRules.yaml
@@ -13,11 +13,21 @@ spec:
   groups:
   - name: alertrules.appes
     rules:
-    - alert: cassandra_node_down_fatal
+    - alert: cassandra_node_down_fatal_Core
       annotations:
         message: cassandra_node_down_fatal
         summary: {{`"There are only {{$value}}`}} < {{ .Values.cassandra_server_count }} {{` Cassandra nodes running"`}}
-      expr: cassandra_stats{name="org:apache:cassandra:db:storageservice:livenodes"} < {{ .Values.cassandra_server_count }}
+      expr: cassandra_stats{name="org:apache:cassandra:db:storageservice:livenodes",job="cassandra-exporter"} < {{ .Values.cassandra_server_count }}
       for: 5m
       labels:
         severity: fatal
+    {{- if .Values.report_cassandra_server_count }}
+    - alert: cassandra_node_down_fatal_Report
+      annotations:
+        message: cassandra_node_down_fatal
+        summary: {{`"There are only {{$value}}`}} < {{ .Values.report_cassandra_server_count }} {{` Cassandra nodes running"`}}
+      expr: cassandra_stats{name="org:apache:cassandra:db:storageservice:livenodes",job="report-cassandra-exporter"} < {{ .Values.report_cassandra_server_count }}
+      for: 5m
+      labels:
+        severity: fatal
+    {{- end }}
diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml
@@ -248,3 +248,6 @@ grafana_google_oauth_client_secret: "xxxxxxxxxxxxxxxxxxxxxxxx"
 #   [email protected]
 #   [email protected]
 grafana_login_whitelisted_emails: ""
+
+Add below var to monitor report-cassandra server
+report_cassandra_server_count: "{{ groups['report-cassandra'] | length }}"