Help debug k8s-daemonset config #105
Labels
Comments
|
Just to add some more information about the setup - k8s (v1.4.8) cluster was started through I had similar 502s on 0.8.6 the day before. Here's a 502 example:
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@aalimovs reports that he's seeing 502s when trying to do a simple test against a kubernetes l2l setup on AWS t2.large instances.
Using this configuration: https://github.com/BuoyantIO/linkerd-examples/blob/master/k8s-daemonset/k8s/linkerd-zipkin.yml
metrics.json snapshot
{ "clnt/daemonsetTransformer/available" : 1.0, "clnt/daemonsetTransformer/cancelled_connects" : 0, "clnt/daemonsetTransformer/closes" : 0, "clnt/daemonsetTransformer/connect_latency_ms.count" : 0, "clnt/daemonsetTransformer/connection_duration.count" : 0, "clnt/daemonsetTransformer/connection_received_bytes.count" : 0, "clnt/daemonsetTransformer/connection_requests.count" : 0, "clnt/daemonsetTransformer/connection_sent_bytes.count" : 0, "clnt/daemonsetTransformer/connections" : 1.0, "clnt/daemonsetTransformer/connects" : 1, "clnt/daemonsetTransformer/dispatcher/serial/queue_size" : 0.0, "clnt/daemonsetTransformer/dtab/size.count" : 0, "clnt/daemonsetTransformer/failed_connect_latency_ms.count" : 0, "clnt/daemonsetTransformer/failfast/marked_available" : 0, "clnt/daemonsetTransformer/failfast/marked_dead" : 0, "clnt/daemonsetTransformer/failfast/unhealthy_for_ms" : 0.0, "clnt/daemonsetTransformer/failfast/unhealthy_num_tries" : 0.0, "clnt/daemonsetTransformer/failure_accrual/probes" : 0, "clnt/daemonsetTransformer/failure_accrual/removals" : 0, "clnt/daemonsetTransformer/failure_accrual/removed_for_ms" : 0, "clnt/daemonsetTransformer/failure_accrual/revivals" : 0, "clnt/daemonsetTransformer/loadbalancer/adds" : 1, "clnt/daemonsetTransformer/loadbalancer/available" : 1.0, "clnt/daemonsetTransformer/loadbalancer/busy" : 0.0, "clnt/daemonsetTransformer/loadbalancer/closed" : 0.0, "clnt/daemonsetTransformer/loadbalancer/load" : 1.0, "clnt/daemonsetTransformer/loadbalancer/max_effort_exhausted" : 0, "clnt/daemonsetTransformer/loadbalancer/meanweight" : 1.0, "clnt/daemonsetTransformer/loadbalancer/p2c" : 1.0, "clnt/daemonsetTransformer/loadbalancer/rebuilds" : 99, "clnt/daemonsetTransformer/loadbalancer/removes" : 0, "clnt/daemonsetTransformer/loadbalancer/size" : 1.0, "clnt/daemonsetTransformer/loadbalancer/updates" : 99, "clnt/daemonsetTransformer/namer/bind_latency_us.count" : 0, "clnt/daemonsetTransformer/namer/dtabcache/evicts" : 0, "clnt/daemonsetTransformer/namer/dtabcache/misses" : 1, "clnt/daemonsetTransformer/namer/dtabcache/oneshots" : 0, "clnt/daemonsetTransformer/namer/namecache/evicts" : 0, "clnt/daemonsetTransformer/namer/namecache/misses" : 1, "clnt/daemonsetTransformer/namer/namecache/oneshots" : 0, "clnt/daemonsetTransformer/namer/nametreecache/evicts" : 0, "clnt/daemonsetTransformer/namer/nametreecache/misses" : 1, "clnt/daemonsetTransformer/namer/nametreecache/oneshots" : 0, "clnt/daemonsetTransformer/pending" : 0.0, "clnt/daemonsetTransformer/pool_cached" : 0.0, "clnt/daemonsetTransformer/pool_num_too_many_waiters" : 0, "clnt/daemonsetTransformer/pool_num_waited" : 0, "clnt/daemonsetTransformer/pool_size" : 1.0, "clnt/daemonsetTransformer/pool_waiters" : 0.0, "clnt/daemonsetTransformer/received_bytes" : 4557, "clnt/daemonsetTransformer/request_latency_ms.count" : 0, "clnt/daemonsetTransformer/requests" : 2, "clnt/daemonsetTransformer/retries/budget" : 100.0, "clnt/daemonsetTransformer/retries/budget_exhausted" : 0, "clnt/daemonsetTransformer/retries/cannot_retry" : 0, "clnt/daemonsetTransformer/retries/not_open" : 0, "clnt/daemonsetTransformer/retries/request_limit" : 0, "clnt/daemonsetTransformer/retries/requeues" : 0, "clnt/daemonsetTransformer/retries/requeues_per_request.count" : 0, "clnt/daemonsetTransformer/sent_bytes" : 491, "clnt/daemonsetTransformer/service_creation/service_acquisition_latency_ms.count" : 0, "clnt/daemonsetTransformer/socket_unwritable_ms" : 0, "clnt/daemonsetTransformer/socket_writable_ms" : 0, "clnt/daemonsetTransformer/success" : 2, "clnt/zipkin-tracer/loadbalancer/adds" : 1, "clnt/zipkin-tracer/loadbalancer/available" : 1.0, "clnt/zipkin-tracer/loadbalancer/busy" : 0.0, "clnt/zipkin-tracer/loadbalancer/closed" : 0.0, "clnt/zipkin-tracer/loadbalancer/load" : 0.0, "clnt/zipkin-tracer/loadbalancer/max_effort_exhausted" : 0, "clnt/zipkin-tracer/loadbalancer/meanweight" : 1.0, "clnt/zipkin-tracer/loadbalancer/p2c" : 1.0, "clnt/zipkin-tracer/loadbalancer/rebuilds" : 1, "clnt/zipkin-tracer/loadbalancer/removes" : 0, "clnt/zipkin-tracer/loadbalancer/size" : 1.0, "clnt/zipkin-tracer/loadbalancer/updates" : 1, "clnt/zipkin-tracer/retries/budget" : 100.0, "clnt/zipkin-tracer/retries/budget_exhausted" : 0, "clnt/zipkin-tracer/retries/cannot_retry" : 0, "clnt/zipkin-tracer/retries/not_open" : 0, "clnt/zipkin-tracer/retries/request_limit" : 0, "clnt/zipkin-tracer/retries/requeues" : 0, "clnt/zipkin-tracer/retries/requeues_per_request.count" : 0, "clnt/zipkin-tracer/service_creation/service_acquisition_latency_ms.count" : 0, "clnt/zipkin-tracer/tries/pending" : 0.0, "clnt/zipkin-tracer/tries/request_latency_ms.count" : 0, "clnt/zipkin-tracer/tries/requests" : 0, "clnt/zipkin-tracer/tries/success" : 0, "inet/dns/cache/evicts" : 0.0, "inet/dns/cache/hit_rate" : 1.0, "inet/dns/cache/size" : 0.0, "inet/dns/dns_lookup_failures" : 0, "inet/dns/dns_lookups" : 0, "inet/dns/failures" : 0, "inet/dns/lookup_ms.count" : 48, "inet/dns/lookup_ms.max" : 0, "inet/dns/lookup_ms.min" : 0, "inet/dns/lookup_ms.p50" : 0, "inet/dns/lookup_ms.p90" : 0, "inet/dns/lookup_ms.p95" : 0, "inet/dns/lookup_ms.p99" : 0, "inet/dns/lookup_ms.p9990" : 0, "inet/dns/lookup_ms.p9999" : 0, "inet/dns/lookup_ms.sum" : 0, "inet/dns/lookup_ms.avg" : 0.0, "inet/dns/queue_size" : 0.0, "inet/dns/successes" : 388, "jvm/application_time_millis" : 494709.0, "jvm/classes/current_loaded" : 7280.0, "jvm/classes/total_loaded" : 7280.0, "jvm/classes/total_unloaded" : 0.0, "jvm/compilation/time_msec" : 2722.0, "jvm/fd_count" : 42.0, "jvm/fd_limit" : 1048576.0, "jvm/gc/ConcurrentMarkSweep/cycles" : 2.0, "jvm/gc/ConcurrentMarkSweep/msec" : 22.0, "jvm/gc/ParNew/cycles" : 32.0, "jvm/gc/ParNew/msec" : 126.0, "jvm/gc/cycles" : 34.0, "jvm/gc/eden/pause_msec.count" : 3, "jvm/gc/eden/pause_msec.max" : 2, "jvm/gc/eden/pause_msec.min" : 2, "jvm/gc/eden/pause_msec.p50" : 2, "jvm/gc/eden/pause_msec.p90" : 2, "jvm/gc/eden/pause_msec.p95" : 2, "jvm/gc/eden/pause_msec.p99" : 2, "jvm/gc/eden/pause_msec.p9990" : 2, "jvm/gc/eden/pause_msec.p9999" : 2, "jvm/gc/eden/pause_msec.sum" : 6, "jvm/gc/eden/pause_msec.avg" : 2.0, "jvm/gc/msec" : 148.0, "jvm/heap/committed" : 3.244032E7, "jvm/heap/max" : 1.05630925E9, "jvm/heap/used" : 1.8293512E7, "jvm/mem/allocations/eden/bytes" : 1.40168416E8, "jvm/mem/buffer/direct/count" : 8.0, "jvm/mem/buffer/direct/max" : 266240.0, "jvm/mem/buffer/direct/used" : 266240.0, "jvm/mem/buffer/mapped/count" : 0.0, "jvm/mem/buffer/mapped/max" : 0.0, "jvm/mem/buffer/mapped/used" : 0.0, "jvm/mem/current/CMS_Old_Gen/max" : 8.9928499E8, "jvm/mem/current/CMS_Old_Gen/used" : 1.1704056E7, "jvm/mem/current/Code_Cache/max" : 5.0331648E7, "jvm/mem/current/Code_Cache/used" : 1737792.0, "jvm/mem/current/Compressed_Class_Space/max" : 1.07374182E9, "jvm/mem/current/Compressed_Class_Space/used" : 6411144.0, "jvm/mem/current/Metaspace/max" : -1.0, "jvm/mem/current/Metaspace/used" : 3.6028464E7, "jvm/mem/current/Par_Eden_Space/max" : 1.3959168E8, "jvm/mem/current/Par_Eden_Space/used" : 6266520.0, "jvm/mem/current/Par_Survivor_Space/max" : 1.7432576E7, "jvm/mem/current/Par_Survivor_Space/used" : 327840.0, "jvm/mem/current/used" : 6.2500528E7, "jvm/mem/metaspace/max_capacity" : 1.0968105E9, "jvm/mem/postGC/CMS_Old_Gen/max" : 8.9928499E8, "jvm/mem/postGC/CMS_Old_Gen/used" : 8868664.0, "jvm/mem/postGC/Par_Eden_Space/max" : 1.3959168E8, "jvm/mem/postGC/Par_Eden_Space/used" : 0.0, "jvm/mem/postGC/Par_Survivor_Space/max" : 1.7432576E7, "jvm/mem/postGC/Par_Survivor_Space/used" : 327840.0, "jvm/mem/postGC/used" : 9196504.0, "jvm/nonheap/committed" : 4.5699072E7, "jvm/nonheap/max" : -1.0, "jvm/nonheap/used" : 4.4191576E7, "jvm/num_cpus" : 2.0, "jvm/postGC/CMS_Old_Gen/max" : 8.9928499E8, "jvm/postGC/CMS_Old_Gen/used" : 8868664.0, "jvm/postGC/Par_Eden_Space/max" : 1.3959168E8, "jvm/postGC/Par_Eden_Space/used" : 0.0, "jvm/postGC/Par_Survivor_Space/max" : 1.7432576E7, "jvm/postGC/Par_Survivor_Space/used" : 327840.0, "jvm/postGC/used" : 9196504.0, "jvm/safepoint/count" : 510.0, "jvm/safepoint/sync_time_millis" : 84.0, "jvm/safepoint/total_time_millis" : 266.0, "jvm/start_time" : 1.48795372E12, "jvm/tenuring_threshold" : 6.0, "jvm/thread/count" : 17.0, "jvm/thread/daemon_count" : 16.0, "jvm/thread/peak_count" : 19.0, "jvm/uptime" : 495175.0, "larger_than_threadlocal_out_buffer" : 0, "namer/namer/#/io.l5d.k8s/default/available" : 1.0, "namer/namer/#/io.l5d.k8s/default/cancelled_connects" : 0, "namer/namer/#/io.l5d.k8s/default/closes" : 0, "namer/namer/#/io.l5d.k8s/default/connect_latency_ms.count" : 0, "namer/namer/#/io.l5d.k8s/default/connection_duration.count" : 0, "namer/namer/#/io.l5d.k8s/default/connection_received_bytes.count" : 0, "namer/namer/#/io.l5d.k8s/default/connection_requests.count" : 0, "namer/namer/#/io.l5d.k8s/default/connection_sent_bytes.count" : 0, "namer/namer/#/io.l5d.k8s/default/connections" : 1.0, "namer/namer/#/io.l5d.k8s/default/connects" : 1, "namer/namer/#/io.l5d.k8s/default/dispatcher/serial/queue_size" : 0.0, "namer/namer/#/io.l5d.k8s/default/dtab/size.count" : 0, "namer/namer/#/io.l5d.k8s/default/failed_connect_latency_ms.count" : 0, "namer/namer/#/io.l5d.k8s/default/failfast/marked_available" : 0, "namer/namer/#/io.l5d.k8s/default/failfast/marked_dead" : 0, "namer/namer/#/io.l5d.k8s/default/failfast/unhealthy_for_ms" : 0.0, "namer/namer/#/io.l5d.k8s/default/failfast/unhealthy_num_tries" : 0.0, "namer/namer/#/io.l5d.k8s/default/failure_accrual/probes" : 0, "namer/namer/#/io.l5d.k8s/default/failure_accrual/removals" : 0, "namer/namer/#/io.l5d.k8s/default/failure_accrual/removed_for_ms" : 0, "namer/namer/#/io.l5d.k8s/default/failure_accrual/revivals" : 0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/adds" : 1, "namer/namer/#/io.l5d.k8s/default/loadbalancer/available" : 1.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/busy" : 0.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/closed" : 0.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/load" : 1.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/max_effort_exhausted" : 0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/meanweight" : 1.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/p2c" : 1.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/rebuilds" : 95, "namer/namer/#/io.l5d.k8s/default/loadbalancer/removes" : 0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/size" : 1.0, "namer/namer/#/io.l5d.k8s/default/loadbalancer/updates" : 95, "namer/namer/#/io.l5d.k8s/default/namer/bind_latency_us.count" : 0, "namer/namer/#/io.l5d.k8s/default/namer/dtabcache/evicts" : 0, "namer/namer/#/io.l5d.k8s/default/namer/dtabcache/misses" : 1, "namer/namer/#/io.l5d.k8s/default/namer/dtabcache/oneshots" : 0, "namer/namer/#/io.l5d.k8s/default/namer/namecache/evicts" : 0, "namer/namer/#/io.l5d.k8s/default/namer/namecache/misses" : 1, "namer/namer/#/io.l5d.k8s/default/namer/namecache/oneshots" : 0, "namer/namer/#/io.l5d.k8s/default/namer/nametreecache/evicts" : 0, "namer/namer/#/io.l5d.k8s/default/namer/nametreecache/misses" : 1, "namer/namer/#/io.l5d.k8s/default/namer/nametreecache/oneshots" : 0, "namer/namer/#/io.l5d.k8s/default/pending" : 0.0, "namer/namer/#/io.l5d.k8s/default/pool_cached" : 0.0, "namer/namer/#/io.l5d.k8s/default/pool_num_too_many_waiters" : 0, "namer/namer/#/io.l5d.k8s/default/pool_num_waited" : 0, "namer/namer/#/io.l5d.k8s/default/pool_size" : 1.0, "namer/namer/#/io.l5d.k8s/default/pool_waiters" : 0.0, "namer/namer/#/io.l5d.k8s/default/received_bytes" : 3654, "namer/namer/#/io.l5d.k8s/default/request_latency_ms.count" : 0, "namer/namer/#/io.l5d.k8s/default/requests" : 2, "namer/namer/#/io.l5d.k8s/default/retries/budget" : 100.0, "namer/namer/#/io.l5d.k8s/default/retries/budget_exhausted" : 0, "namer/namer/#/io.l5d.k8s/default/retries/cannot_retry" : 0, "namer/namer/#/io.l5d.k8s/default/retries/not_open" : 0, "namer/namer/#/io.l5d.k8s/default/retries/request_limit" : 0, "namer/namer/#/io.l5d.k8s/default/retries/requeues" : 0, "namer/namer/#/io.l5d.k8s/default/retries/requeues_per_request.count" : 0, "namer/namer/#/io.l5d.k8s/default/sent_bytes" : 491, "namer/namer/#/io.l5d.k8s/default/service_creation/service_acquisition_latency_ms.count" : 0, "namer/namer/#/io.l5d.k8s/default/socket_unwritable_ms" : 0, "namer/namer/#/io.l5d.k8s/default/socket_writable_ms" : 0, "namer/namer/#/io.l5d.k8s/default/success" : 2, "retries.count" : 0, "retries/budget_exhausted" : 0, "rt/incoming/bindcache/bound/evicts" : 0, "rt/incoming/bindcache/bound/misses" : 0, "rt/incoming/bindcache/bound/oneshots" : 0, "rt/incoming/bindcache/client/evicts" : 0, "rt/incoming/bindcache/client/misses" : 0, "rt/incoming/bindcache/client/oneshots" : 0, "rt/incoming/bindcache/path/evicts" : 0, "rt/incoming/bindcache/path/misses" : 0, "rt/incoming/bindcache/path/oneshots" : 0, "rt/incoming/bindcache/tree/evicts" : 0, "rt/incoming/bindcache/tree/misses" : 0, "rt/incoming/bindcache/tree/oneshots" : 0, "rt/incoming/srv/0.0.0.0/4141/dtab/size.count" : 0, "rt/incoming/srv/0.0.0.0/4141/handletime_us.count" : 0, "rt/incoming/srv/0.0.0.0/4141/nacks" : 0, "rt/incoming/srv/0.0.0.0/4141/nonretryable_nacks" : 0, "rt/incoming/srv/0.0.0.0/4141/pending" : 0.0, "rt/incoming/srv/0.0.0.0/4141/request_latency_ms.count" : 0, "rt/incoming/srv/0.0.0.0/4141/requests" : 0, "rt/incoming/srv/0.0.0.0/4141/status/1XX" : 0, "rt/incoming/srv/0.0.0.0/4141/status/2XX" : 0, "rt/incoming/srv/0.0.0.0/4141/status/3XX" : 0, "rt/incoming/srv/0.0.0.0/4141/status/4XX" : 0, "rt/incoming/srv/0.0.0.0/4141/status/5XX" : 0, "rt/incoming/srv/0.0.0.0/4141/status/error" : 0, "rt/incoming/srv/0.0.0.0/4141/success" : 0, "rt/incoming/srv/0.0.0.0/4141/time/1XX.count" : 0, "rt/incoming/srv/0.0.0.0/4141/time/2XX.count" : 0, "rt/incoming/srv/0.0.0.0/4141/time/3XX.count" : 0, "rt/incoming/srv/0.0.0.0/4141/time/4XX.count" : 0, "rt/incoming/srv/0.0.0.0/4141/time/5XX.count" : 0, "rt/incoming/srv/0.0.0.0/4141/time/error.count" : 0, "rt/incoming/srv/0.0.0.0/4141/transit_latency_ms.count" : 0, "rt/outgoing/bindcache/bound/evicts" : 0, "rt/outgoing/bindcache/bound/misses" : 1, "rt/outgoing/bindcache/bound/oneshots" : 0, "rt/outgoing/bindcache/client/evicts" : 0, "rt/outgoing/bindcache/client/misses" : 1, "rt/outgoing/bindcache/client/oneshots" : 0, "rt/outgoing/bindcache/path/evicts" : 0, "rt/outgoing/bindcache/path/misses" : 2, "rt/outgoing/bindcache/path/oneshots" : 0, "rt/outgoing/bindcache/tree/evicts" : 0, "rt/outgoing/bindcache/tree/misses" : 2, "rt/outgoing/bindcache/tree/oneshots" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/available" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/cancelled_connects" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/closes" : 2342, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connect_latency_ms.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.max" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.min" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p50" : 63, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p90" : 329, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p95" : 547, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p99" : 680, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p9990" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.p9999" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.sum" : 57355, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_duration.avg" : 115.86868686868686, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.max" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.min" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p50" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p90" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p95" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p99" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p9990" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.p9999" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.sum" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_received_bytes.avg" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.max" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.min" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p50" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p90" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p95" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p99" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p9990" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.p9999" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.sum" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_requests.avg" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.max" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.min" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p50" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p90" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p95" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p99" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p9990" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.p9999" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.sum" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connection_sent_bytes.avg" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connections" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/connects" : 1171, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/dtab/size.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/exn/org.jboss.netty.channel.ConnectTimeoutException" : 1171, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.max" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.min" : 1003, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p50" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p90" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p95" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p99" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p9990" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.p9999" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.sum" : 499282, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failed_connect_latency_ms.avg" : 1008.6505050505051, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failure_accrual/probes" : 5, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failure_accrual/removals" : 1, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failure_accrual/removed_for_ms" : 228371, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/failure_accrual/revivals" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/adds" : 1, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/available" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/busy" : 1.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/closed" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/load" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/max_effort_exhausted" : 1153, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/meanweight" : 1.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/p2c" : 1.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/rebuilds" : 1154, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/removes" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/size" : 1.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/loadbalancer/updates" : 1, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pending" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pool_cached" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pool_num_too_many_waiters" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pool_num_waited" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pool_size" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/pool_waiters" : 0.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/received_bytes" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/request_latency_ms.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/requests" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/budget" : 100.0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/budget_exhausted" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/cannot_retry" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/not_open" : 1143, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/request_limit" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/requeues" : 4, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/retries/requeues_per_request.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/sent_bytes" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/failures" : 1171, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/failures/com.twitter.finagle.ChannelWriteException" : 1147, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/failures/com.twitter.finagle.ChannelWriteException/org.jboss.netty.channel.ConnectTimeoutException" : 1147, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/failures/com.twitter.finagle.Failure" : 24, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/failures/com.twitter.finagle.Failure/com.twitter.finagle.CancelledRequestException" : 24, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.count" : 495, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.max" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.min" : 152, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p50" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p90" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p95" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p99" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p9990" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.p9999" : 1013, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.sum" : 494421, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/service_creation/service_acquisition_latency_ms.avg" : 998.8303030303031, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/socket_unwritable_ms" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/socket_writable_ms" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/1XX" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/2XX" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/3XX" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/4XX" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/5XX" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/status/error" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/success" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/1XX.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/2XX.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/3XX.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/4XX.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/5XX.count" : 0, "rt/outgoing/dst/id/%/io.l5d.k8s.daemonset/default/incoming/l5d/#/io.l5d.k8s/default/http/nginx/time/error.count" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/failures" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/failures/com.twitter.finagle.NoBrokersAvailableException" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/pending" : 0.0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.count" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.max" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.min" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p50" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p90" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p95" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p99" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p9990" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.p9999" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.sum" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/request_latency_ms.avg" : 0.0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/requests" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/budget_exhausted" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.count" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.max" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.min" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p50" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p90" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p95" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p99" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p9990" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.p9999" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.sum" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/per_request.avg" : 0.0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/retries/total" : 0, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/sourcedfailures//svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/sourcedfailures//svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/com.twitter.finagle.NoBrokersAvailableException" : 18, "rt/outgoing/dst/path/svc/aeb6fe752fa9711e6a23906490c9cdb1-246420995.eu-west-1.elb.amazonaws.com:4140/success" : 0, "rt/outgoing/dst/path/svc/nginx/failures" : 46, "rt/outgoing/dst/path/svc/nginx/failures/com.twitter.finagle.ChannelWriteException" : 22, "rt/outgoing/dst/path/svc/nginx/failures/com.twitter.finagle.ChannelWriteException/org.jboss.netty.channel.ConnectTimeoutException" : 22, "rt/outgoing/dst/path/svc/nginx/failures/interrupted" : 24, "rt/outgoing/dst/path/svc/nginx/failures/interrupted/com.twitter.finagle.Failure" : 24, "rt/outgoing/dst/path/svc/nginx/failures/interrupted/com.twitter.finagle.Failure/com.twitter.finagle.CancelledRequestException" : 24, "rt/outgoing/dst/path/svc/nginx/failures/restartable" : 24, "rt/outgoing/dst/path/svc/nginx/failures/restartable/com.twitter.finagle.Failure" : 24, "rt/outgoing/dst/path/svc/nginx/failures/restartable/com.twitter.finagle.Failure/com.twitter.finagle.CancelledRequestException" : 24, "rt/outgoing/dst/path/svc/nginx/pending" : 0.0, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.count" : 37, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.max" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.min" : 1003, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p50" : 22360, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p90" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p95" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p99" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p9990" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.p9999" : 30138, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.sum" : 641204, "rt/outgoing/dst/path/svc/nginx/request_latency_ms.avg" : 17329.837837837837, "rt/outgoing/dst/path/svc/nginx/requests" : 46, "rt/outgoing/dst/path/svc/nginx/retries/budget_exhausted" : 22, "rt/outgoing/dst/path/svc/nginx/retries/per_request.count" : 37, "rt/outgoing/dst/path/svc/nginx/retries/per_request.max" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.min" : 0, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p50" : 22, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p90" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p95" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p99" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p9990" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.p9999" : 29, "rt/outgoing/dst/path/svc/nginx/retries/per_request.sum" : 603, "rt/outgoing/dst/path/svc/nginx/retries/per_request.avg" : 16.2972972972973, "rt/outgoing/dst/path/svc/nginx/retries/total" : 1121, "rt/outgoing/dst/path/svc/nginx/success" : 0, "rt/outgoing/srv/0.0.0.0/4140/closes" : 0, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.count" : 36, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.max" : 11595, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.min" : 0, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p50" : 161, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p90" : 7264, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p95" : 9694, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p99" : 11595, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p9990" : 11595, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.p9999" : 11595, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.sum" : 83092, "rt/outgoing/srv/0.0.0.0/4140/connection_duration.avg" : 2308.1111111111113, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.count" : 36, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.max" : 480, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.min" : 0, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p50" : 139, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p90" : 386, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p95" : 386, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p99" : 480, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p9990" : 480, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.p9999" : 480, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.sum" : 5680, "rt/outgoing/srv/0.0.0.0/4140/connection_received_bytes.avg" : 157.77777777777777, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.count" : 36, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.max" : 5, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.min" : 0, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p50" : 1, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p90" : 4, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p95" : 4, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p99" : 5, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p9990" : 5, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.p9999" : 5, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.sum" : 52, "rt/outgoing/srv/0.0.0.0/4140/connection_requests.avg" : 1.4444444444444444, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.count" : 36, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.max" : 3584, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.min" : 0, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p50" : 654, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p90" : 2686, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p95" : 2686, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p99" : 3584, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p9990" : 3584, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.p9999" : 3584, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.sum" : 29322, "rt/outgoing/srv/0.0.0.0/4140/connection_sent_bytes.avg" : 814.5, "rt/outgoing/srv/0.0.0.0/4140/connections" : 1.0, "rt/outgoing/srv/0.0.0.0/4140/connects" : 65, "rt/outgoing/srv/0.0.0.0/4140/dtab/size.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/exn/java.nio.channels.ClosedChannelException" : 24, "rt/outgoing/srv/0.0.0.0/4140/failures" : 64, "rt/outgoing/srv/0.0.0.0/4140/failures/com.twitter.finagle.service.ResponseClassificationSyntheticException" : 64, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.count" : 44, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.max" : 187, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.min" : 75, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p50" : 118, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p90" : 140, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p95" : 171, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p99" : 187, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p9990" : 187, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.p9999" : 187, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.sum" : 5327, "rt/outgoing/srv/0.0.0.0/4140/handletime_us.avg" : 121.06818181818181, "rt/outgoing/srv/0.0.0.0/4140/nacks" : 0, "rt/outgoing/srv/0.0.0.0/4140/nonretryable_nacks" : 0, "rt/outgoing/srv/0.0.0.0/4140/pending" : 0.0, "rt/outgoing/srv/0.0.0.0/4140/read_timeout" : 0, "rt/outgoing/srv/0.0.0.0/4140/received_bytes" : 7696, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.count" : 55, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.max" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.min" : 1, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p50" : 7050, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p90" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p95" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p99" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p9990" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.p9999" : 30138, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.sum" : 641295, "rt/outgoing/srv/0.0.0.0/4140/request_latency_ms.avg" : 11659.90909090909, "rt/outgoing/srv/0.0.0.0/4140/requests" : 64, "rt/outgoing/srv/0.0.0.0/4140/sent_bytes" : 31418, "rt/outgoing/srv/0.0.0.0/4140/socket_unwritable_ms" : 0, "rt/outgoing/srv/0.0.0.0/4140/socket_writable_ms" : 0, "rt/outgoing/srv/0.0.0.0/4140/status/1XX" : 0, "rt/outgoing/srv/0.0.0.0/4140/status/2XX" : 0, "rt/outgoing/srv/0.0.0.0/4140/status/3XX" : 0, "rt/outgoing/srv/0.0.0.0/4140/status/4XX" : 0, "rt/outgoing/srv/0.0.0.0/4140/status/502" : 64, "rt/outgoing/srv/0.0.0.0/4140/status/5XX" : 64, "rt/outgoing/srv/0.0.0.0/4140/status/error" : 0, "rt/outgoing/srv/0.0.0.0/4140/success" : 0, "rt/outgoing/srv/0.0.0.0/4140/time/1XX.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/time/2XX.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/time/3XX.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/time/4XX.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.count" : 55, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.max" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.min" : 1, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p50" : 7050, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p90" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p95" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p99" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p9990" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.p9999" : 30138, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.sum" : 641325, "rt/outgoing/srv/0.0.0.0/4140/time/5XX.avg" : 11660.454545454546, "rt/outgoing/srv/0.0.0.0/4140/time/error.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/transit_latency_ms.count" : 0, "rt/outgoing/srv/0.0.0.0/4140/write_timeout" : 0, "zipkin/zipkin/log_span/ok" : 0, "zipkin/zipkin/log_span/try_later" : 0, "zk2/inet/dns/cache/evicts" : 0.0, "zk2/inet/dns/cache/hit_rate" : 1.0, "zk2/inet/dns/cache/size" : 0.0, "zk2/inet/dns/dns_lookup_failures" : 0, "zk2/inet/dns/dns_lookups" : 0, "zk2/inet/dns/failures" : 0, "zk2/inet/dns/lookup_ms.count" : 0, "zk2/inet/dns/queue_size" : 0.0, "zk2/inet/dns/successes" : 0, "zk2/observed_serversets" : 0.0, "zk2/session_cache_size" : 0.0 }What especially stands out here is that the client's
/closesmetric is more than twice that of/connects-- I assume that/closescounts failed connections and/connectsdoes not?We have disabled telemetry, usage reporting, etc, with no improvement. Why can't we keep connections established?
The text was updated successfully, but these errors were encountered: