Merge pull request #43 from signalfx/es5-support

Add support for ES 5.0.0

elasticsearch_collectd.py

@@ -303,6 +303,28 @@
         'revision': 0,
         'keys': ['process.mem.share_in_bytes'],
     },
+    {
+        'major': 5,
+        'minor': 0,
+        'revision': 0,
+        'keys': ['indices.segments.index-writer-max-size']
+    }
+]
+
+# Deprecated thread pools by first version in which they were removed
+DEPRECATED_THREAD_POOLS = [
+    {
+        'major': 2,
+        'minor': 0,
+        'revision': 0,
+        'keys': ['merge', 'optimize']
+    },
+    {
+        'major': 5,
+        'minor': 0,
+        'revision': 0,
+        'keys': ['suggest', 'percolate']
+    }
 ]
 
 NODE_STATS_ES_2 = {
@@ -726,21 +748,43 @@ def sanatize_intervals():
 def remove_deprecated_node_stats():
     """Remove deprecated node stats from the list of stats to collect"""
     global DEPRECATED_NODE_STATS, ES_VERSION, NODE_STATS_CUR
+    NODE_STATS_CUR = remove_deprecated_elements(DEPRECATED_NODE_STATS,
+                                                NODE_STATS_CUR,
+                                                ES_VERSION)
+
+
+def remove_deprecated_threads():
+    """Remove deprecated thread_pools from the list of stats to collect"""
+    global DEPRECATED_THREAD_POOLS, ES_VERSION, THREAD_POOLS
+    THREAD_POOLS = remove_deprecated_elements(DEPRECATED_THREAD_POOLS,
+                                              THREAD_POOLS,
+                                              ES_VERSION)
+
+
+def remove_deprecated_elements(deprecated, elements, version):
+    """Remove deprecated items from a list or dictionary"""
     # Attempt to parse the major, minor, and revision
-    (major, minor, revision) = ES_VERSION.split('.')
+    (major, minor, revision) = version.split('.')
 
     # Sanatize alphas and betas from revision number
     revision = revision.split('-')[0]
 
     # Iterate over deprecation lists and remove any keys that were deprecated
     # prior to the current version
-    for dep in DEPRECATED_NODE_STATS:
+    for dep in deprecated:
         if (major >= dep['major']) \
             or (major == dep['major'] and minor >= dep['minor']) \
             or (major == dep['major'] and minor == dep['minor'] and
                 revision >= dep['revision']):
-            for key in dep['keys']:
-                del NODE_STATS_CUR[key]
+            if type(elements) is list:
+                for key in dep['keys']:
+                    if key in elements:
+                        elements.remove(key)
+            if type(elements) is dict:
+                for key in dep['keys']:
+                    if key in elements:
+                        del elements[key]
+    return elements
 
 
 # helper methods
@@ -758,7 +802,7 @@ def init_stats():
                   "thread_pool"
     NODE_STATS_CUR = dict(NODE_STATS.items())
     INDEX_STATS_CUR = dict(INDEX_STATS.items())
-    if ES_VERSION.startswith("2."):
+    if not ES_VERSION.startswith("1."):
         NODE_STATS_CUR.update(NODE_STATS_ES_2)
 
     remove_deprecated_node_stats()
@@ -783,13 +827,18 @@ def init_stats():
     thread_pools = ['generic', 'index', 'get', 'snapshot', 'bulk', 'warmer',
                     'flush', 'search', 'refresh']
 
-    if ES_VERSION.startswith("2."):
+    # Add the 1.0 metrics
+    if not ES_VERSION.startswith("0."):
+        thread_pools.extend(['merge', 'optimize'])
+
+    # Add the 2.0 metrics
+    if not ES_VERSION.startswith("1."):
         thread_pools.extend(['suggest', 'percolate', 'management', 'listener',
                              'fetch_shard_store', 'fetch_shard_started'])
-    elif ES_VERSION.startswith("2.") and not ES_VERSION.startswith("2.0"):
+
+    # Add the 2.1 metrics
+    if not ES_VERSION.startswith("1.") and not ES_VERSION.startswith("2.0"):
         thread_pools.extend(['force_merge'])
-    else:
-        thread_pools.extend(['merge', 'optimize'])
 
     # Legacy support for old configurations without Thread Pools configuration
     if len(CONFIGURED_THREAD_POOLS) == 0:
@@ -799,6 +848,8 @@ def init_stats():
         THREAD_POOLS = filter(lambda pool: pool in CONFIGURED_THREAD_POOLS,
                               thread_pools)
 
+    remove_deprecated_threads()
+
     ES_CLUSTER_URL = "http://" + ES_HOST + \
                      ":" + str(ES_PORT) + "/_cluster/health"
 
@@ -1199,6 +1250,7 @@ def configure_test():
     # allow user to override ES host name for easier testing
     if len(sys.argv) > 1:
         ES_HOST = sys.argv[1]
+    handle.verbose = True
     collectd = CollectdMock()
     configure_test()
     load_es_info()

tests/data/5.0.0/_all/_stats/index.json

@@ -1 +1 @@
-{"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":194,"fixed_bit_set_memory_in_bytes":0,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":194,"fixed_bit_set_memory_in_bytes":0,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"twitter":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":194,"fixed_bit_set_memory_in_bytes":0,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":59,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":20},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":194,"fixed_bit_set_memory_in_bytes":0,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":302},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}
+{"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":160},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":16},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":160},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":16},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"twitter":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":160},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":16},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":56,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":1,"total_time_in_millis":160},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":6,"total_time_in_millis":16},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2588,"terms_memory_in_bytes":1992,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}

tests/data/5.0.0/_cluster/state/master_node.json

@@ -1 +1 @@
-{"cluster_name":"elasticsearch","master_node":"OtaJDRObTwy__kQqqOh9xQ"}
+{"cluster_name":"elasticsearch","master_node":"GqR32MnhS8CfxfQyZUIU8g"}