diff --git a/src/alert_rules/prometheus/prometheus_alerts.yaml b/src/alert_rules/prometheus/prometheus_alerts.yaml index 332e40579..965e17bed 100644 --- a/src/alert_rules/prometheus/prometheus_alerts.yaml +++ b/src/alert_rules/prometheus/prometheus_alerts.yaml @@ -131,10 +131,20 @@ - "alert": "OpenSearchThrottling" "annotations": - "message": "Cluster {{ $labels.cluster }} is throttling. Please optimize queries and indexing patterns or consider scale the application." + "message": "Cluster {{ $labels.cluster }} is throttling. Please review your indexing request rate, index lifecycle or consider scale the application." "summary": "OpenSearch Indexing Throttle" "expr": | sum by (cluster) (opensearch_indices_indexing_is_throttled_bool) > 0 "for": "5m" "labels": "severity": "warning" + + - "alert": "OpenSearchThrottlingTooLong" + "annotations": + "message": "Cluster {{ $labels.cluster }} is throttling for at least two hours. Please review your indexing request rate, index lifecycle or consider scale the application." + "summary": "OpenSearch Indexing Throttle too long" + "expr": | + sum by (cluster) (opensearch_indices_indexing_is_throttled_bool) > 0 + "for": "2h" + "labels": + "severity": "critical" diff --git a/tests/unit/test_alert_rules/test_opensearch_rules.yaml b/tests/unit/test_alert_rules/test_opensearch_rules.yaml index 7a6b8f719..3f635d934 100644 --- a/tests/unit/test_alert_rules/test_opensearch_rules.yaml +++ b/tests/unit/test_alert_rules/test_opensearch_rules.yaml @@ -191,5 +191,20 @@ tests: severity: warning cluster: opensearch-x7zb exp_annotations: - message: "Cluster opensearch-x7zb is throttling. Please optimize queries and indexing patterns or consider scale the application." + message: "Cluster opensearch-x7zb is throttling. Please review your indexing request rate, index lifecycle or consider scale the application." summary: "OpenSearch Indexing Throttle" + + - interval: 1m + input_series: + - series: 'opensearch_indices_indexing_is_throttled_bool{cluster="opensearch-x7zb"}' + values: '1x360' + alert_rule_test: + - eval_time: 2h + alertname: OpenSearchThrottlingTooLong + exp_alerts: + - exp_labels: + severity: critical + cluster: opensearch-x7zb + exp_annotations: + message: "Cluster opensearch-x7zb is throttling for at least two hours. Please review your indexing request rate, index lifecycle or consider scale the application." + summary: "OpenSearch Indexing Throttle too long"