Problem checking json output on Elasticsearch server
mattboston opened this issue · 2 comments
mattboston commented
I'm having an issue where I can't figure out how to the the nodes.ZFI8avwBSZCEF59qI5LQvw.jvm.mem.heap_used_percent element. The problem is the field ZFI8avwBSZCEF59qI5LQvw will be different for each server I hit, and depending how much Elasticsearch cluster scales, I have no way of knowing what it is. How can I make this work?
#./check_http_json.py -H integration-search001 -P 9200 -p "_nodes/integration-search001/stats" -f _ -e "nodes_(0)_jvm.mem.heap_used_percent" -d
url:http://integration-search001:9200/_nodes/integration-search001/stats
json:
{u'_nodes': {u'failed': 0, u'successful': 1, u'total': 1},
u'cluster_name': u'integration-search',
u'nodes': {u'ZFI8avwBSZCEF59qI5LQvw': {u'attributes': {u'ml.enabled': u'true'},
u'breakers': {u'fielddata': {u'estimated_size': u'1.5kb',
u'estimated_size_in_bytes': 1560,
u'limit_size': u'2.3gb',
u'limit_size_in_bytes': 2566520832,
u'overhead': 1.03,
u'tripped': 0},
u'in_flight_requests': {u'estimated_size': u'0b',
u'estimated_size_in_bytes': 0,
u'limit_size': u'3.9gb',
u'limit_size_in_bytes': 4277534720,
u'overhead': 1.0,
u'tripped': 0},
u'parent': {u'estimated_size': u'1.5kb',
u'estimated_size_in_bytes': 1560,
u'limit_size': u'2.7gb',
u'limit_size_in_bytes': 2994274304,
u'overhead': 1.0,
u'tripped': 0},
u'request': {u'estimated_size': u'0b',
u'estimated_size_in_bytes': 0,
u'limit_size': u'2.3gb',
u'limit_size_in_bytes': 2566520832,
u'overhead': 1.0,
u'tripped': 0}},
u'discovery': {u'cluster_state_queue': {u'committed': 0,
u'pending': 0,
u'total': 0}},
u'fs': {u'data': [{u'available_in_bytes': 94561394688,
u'free_in_bytes': 99946668032,
u'mount': u'/var/lib/elasticsearch (/dev/mapper/elastic-elastic)',
u'path': u'/var/lib/elasticsearch/integration-search001/nodes/0',
u'spins': u'false',
u'total_in_bytes': 105551003648,
u'type': u'ext4'}],
u'io_stats': {u'devices': [{u'device_name': u'dm-0',
u'operations': 158156893,
u'read_kilobytes': 8456332,
u'read_operations': 83477,
u'write_kilobytes': 2262981964,
u'write_operations': 158073416}],
u'total': {u'operations': 158156893,
u'read_kilobytes': 8456332,
u'read_operations': 83477,
u'write_kilobytes': 2262981964,
u'write_operations': 158073416}},
u'timestamp': 1507212261634,
u'total': {u'available_in_bytes': 94561394688,
u'free_in_bytes': 99946668032,
u'total_in_bytes': 105551003648}},
u'host': u'10.10.193.88',
u'http': {u'current_open': 14,
u'total_opened': 305250},
u'indices': {u'completion': {u'size_in_bytes': 0},
u'docs': {u'count': 21387902,
u'deleted': 402641},
u'fielddata': {u'evictions': 0,
u'memory_size_in_bytes': 1560},
u'flush': {u'total': 205,
u'total_time_in_millis': 21076},
u'get': {u'current': 0,
u'exists_time_in_millis': 0,
u'exists_total': 0,
u'missing_time_in_millis': 0,
u'missing_total': 0,
u'time_in_millis': 0,
u'total': 0},
u'indexing': {u'delete_current': 0,
u'delete_time_in_millis': 0,
u'delete_total': 0,
u'index_current': 0,
u'index_failed': 0,
u'index_time_in_millis': 22506919,
u'index_total': 42364721,
u'is_throttled': False,
u'noop_update_total': 0,
u'throttle_time_in_millis': 0},
u'merges': {u'current': 0,
u'current_docs': 0,
u'current_size_in_bytes': 0,
u'total': 617509,
u'total_auto_throttle_in_bytes': 2272534385,
u'total_docs': 9899704139,
u'total_size_in_bytes': 2400154791963,
u'total_stopped_time_in_millis': 0,
u'total_throttled_time_in_millis': 55731,
u'total_time_in_millis': 1434961775},
u'query_cache': {u'cache_count': 2795,
u'cache_size': 2715,
u'evictions': 80,
u'hit_count': 14499,
u'memory_size_in_bytes': 19427570,
u'miss_count': 82309,
u'total_count': 96808},
u'recovery': {u'current_as_source': 0,
u'current_as_target': 0,
u'throttle_time_in_millis': 1085},
u'refresh': {u'listeners': 0,
u'total': 6008781,
u'total_time_in_millis': 97187070},
u'request_cache': {u'evictions': 0,
u'hit_count': 15099,
u'memory_size_in_bytes': 4662964,
u'miss_count': 4056},
u'search': {u'fetch_current': 0,
u'fetch_time_in_millis': 1437,
u'fetch_total': 84,
u'open_contexts': 0,
u'query_current': 0,
u'query_time_in_millis': 81129,
u'query_total': 38204,
u'scroll_current': 0,
u'scroll_time_in_millis': 0,
u'scroll_total': 0,
u'suggest_current': 0,
u'suggest_time_in_millis': 0,
u'suggest_total': 0},
u'segments': {u'count': 505,
u'doc_values_memory_in_bytes': 7935196,
u'file_sizes': {},
u'fixed_bit_set_memory_in_bytes': 2635568,
u'index_writer_memory_in_bytes': 0,
u'max_unsafe_auto_id_timestamp': 1507161603687,
u'memory_in_bytes': 19966429,
u'norms_memory_in_bytes': 87296,
u'points_memory_in_bytes': 888639,
u'stored_fields_memory_in_bytes': 1928192,
u'term_vectors_memory_in_bytes': 0,
u'terms_memory_in_bytes': 9127106,
u'version_map_memory_in_bytes': 0},
u'store': {u'size_in_bytes': 5513072482,
u'throttle_time_in_millis': 0},
u'translog': {u'operations': 20262,
u'size_in_bytes': 18824115},
u'warmer': {u'current': 0,
u'total': 2585213,
u'total_time_in_millis': 2277747}},
u'ingest': {u'pipelines': {u'xpack_monitoring_2': {u'count': 0,
u'current': 0,
u'failed': 0,
u'time_in_millis': 0},
u'xpack_monitoring_6': {u'count': 0,
u'current': 0,
u'failed': 0,
u'time_in_millis': 0}},
u'total': {u'count': 0,
u'current': 0,
u'failed': 0,
u'time_in_millis': 0}},
u'ip': u'10.10.193.88:9300',
u'jvm': {u'buffer_pools': {u'direct': {u'count': 53,
u'total_capacity_in_bytes': 68183017,
u'used_in_bytes': 68183018},
u'mapped': {u'count': 1294,
u'total_capacity_in_bytes': 5500104160,
u'used_in_bytes': 5500104160}},
u'classes': {u'current_loaded_count': 14327,
u'total_loaded_count': 14388,
u'total_unloaded_count': 61},
u'gc': {u'collectors': {u'old': {u'collection_count': 1584,
u'collection_time_in_millis': 117924},
u'young': {u'collection_count': 725521,
u'collection_time_in_millis': 20444535}}},
u'mem': {u'heap_committed_in_bytes': 4277534720,
u'heap_max_in_bytes': 4277534720,
u'heap_used_in_bytes': 2124974840,
u'heap_used_percent': 49,
u'non_heap_committed_in_bytes': 170213376,
u'non_heap_used_in_bytes': 161598824,
u'pools': {u'old': {u'max_in_bytes': 4120510464,
u'peak_max_in_bytes': 4120510464,
u'peak_used_in_bytes': 4079527400,
u'used_in_bytes': 2115473832},
u'survivor': {u'max_in_bytes': 17432576,
u'peak_max_in_bytes': 17432576,
u'peak_used_in_bytes': 17432576,
u'used_in_bytes': 1253136},
u'young': {u'max_in_bytes': 139591680,
u'peak_max_in_bytes': 139591680,
u'peak_used_in_bytes': 139591680,
u'used_in_bytes': 8247872}}},
u'threads': {u'count': 77,
u'peak_count': 101},
u'timestamp': 1507212261633,
u'uptime_in_millis': 2407678598},
u'name': u'integration-search001',
u'os': {u'cpu': {u'load_average': {u'15m': 0.0,
u'1m': 0.0,
u'5m': 0.0},
u'percent': 5},
u'mem': {u'free_in_bytes': 398741504,
u'free_percent': 5,
u'total_in_bytes': 8373026816,
u'used_in_bytes': 7974285312,
u'used_percent': 95},
u'swap': {u'free_in_bytes': 0,
u'total_in_bytes': 0,
u'used_in_bytes': 0},
u'timestamp': 1507212261633},
u'process': {u'cpu': {u'percent': 4,
u'total_in_millis': 531482350},
u'max_file_descriptors': 65536,
u'mem': {u'total_virtual_in_bytes': 12712378368},
u'open_file_descriptors': 357,
u'timestamp': 1507212261633},
u'roles': [u'master',
u'data',
u'ingest'],
u'script': {u'cache_evictions': 0,
u'compilations': 1},
u'thread_pool': {u'bulk': {u'active': 0,
u'completed': 7524444,
u'largest': 2,
u'queue': 0,
u'rejected': 0,
u'threads': 2},
u'fetch_shard_started': {u'active': 0,
u'completed': 23,
u'largest': 4,
u'queue': 0,
u'rejected': 0,
u'threads': 1},
u'fetch_shard_store': {u'active': 0,
u'completed': 23,
u'largest': 4,
u'queue': 0,
u'rejected': 0,
u'threads': 1},
u'flush': {u'active': 0,
u'completed': 355,
u'largest': 1,
u'queue': 0,
u'rejected': 0,
u'threads': 1},
u'force_merge': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'generic': {u'active': 0,
u'completed': 3130592,
u'largest': 7,
u'queue': 0,
u'rejected': 0,
u'threads': 7},
u'get': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'index': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'listener': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'management': {u'active': 1,
u'completed': 11584744,
u'largest': 5,
u'queue': 0,
u'rejected': 0,
u'threads': 5},
u'ml_autodetect': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'ml_datafeed': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'ml_utility': {u'active': 0,
u'completed': 28,
u'largest': 28,
u'queue': 0,
u'rejected': 0,
u'threads': 28},
u'refresh': {u'active': 0,
u'completed': 21598424,
u'largest': 1,
u'queue': 0,
u'rejected': 0,
u'threads': 1},
u'search': {u'active': 0,
u'completed': 995604,
u'largest': 4,
u'queue': 0,
u'rejected': 0,
u'threads': 4},
u'snapshot': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0},
u'warmer': {u'active': 0,
u'completed': 52166566,
u'largest': 1,
u'queue': 0,
u'rejected': 0,
u'threads': 1},
u'watcher': {u'active': 0,
u'completed': 0,
u'largest': 0,
u'queue': 0,
u'rejected': 0,
u'threads': 0}},
u'timestamp': 1507212261505,
u'transport': {u'rx_count': 53201865,
u'rx_size_in_bytes': 216656566638,
u'server_open': 39,
u'tx_count': 53201865,
u'tx_size_in_bytes': 287894015241},
u'transport_address': u'10.10.193.88:9300'}}}
rules:Namespace(auth=None, data=None, debug=True, headers=None, host='integration-search001', key_list=['nodes_(0)_jvm.mem.heap_used_percent'], key_list_critical=None, key_threshold_critical=None, key_threshold_warning=None, key_value_list=None, key_value_list_critical=None, metric_list=None, path='_nodes/integration-search001/stats', port='9200', separator='_', ssl=False, timeout=None)
separator:_
Traceback (most recent call last):
File "./check_http_json.py", line 437, in <module>
nagios.append_warning(processor.checkWarning())
File "./check_http_json.py", line 214, in checkWarning
failure += self.checkExists(self.rules.key_list)
File "./check_http_json.py", line 154, in checkExists
if (self.helper.exists(key) == False):
File "./check_http_json.py", line 104, in exists
def exists(self, key): return (self.get(key) != (None, 'not_found'))
File "./check_http_json.py", line 115, in get
return self.getSubElement(key, data)
File "./check_http_json.py", line 78, in getSubElement
return self.get(remainingKey, data[partialKey])
File "./check_http_json.py", line 117, in get
return self.getSubArrayElement(key, data)
File "./check_http_json.py", line 95, in getSubArrayElement
return self.get(remainingKey, data[index])
KeyError: 0
martialblog commented
Ok, so from what I can tell the issue is that u'nodes': {u'ZFI8avwBSZCEF59qI5LQvw': is not an array. Therefor the (0) syntax won't work.
Does this error still persist?
martialblog commented
Closed due to inactivity