namerd, high memory usage #1361
Description
Hello, I am seeing high memory usage on some namerd instances.
I am using version 1.0.2
I run namerd in kubernetes on aws, in a daemonset.
All of the namerd pods seem to reproduce the same behaviour, at seemingly random times.
What happens is that namerd will start using ~= 1.1 Gb of memory (it is probably capped by the MaxHeapSize=1073741824 jvm option), and then crash.
namerd outputs this log before crashing :
VM error: Java heap space
java.lang.OutOfMemoryError: Java heap space
at java.util.LinkedHashMap.newNode(LinkedHashMap.java:256)
at java.util.HashMap.putVal(HashMap.java:641)
at java.util.HashMap.put(HashMap.java:611)
at sun.util.resources.OpenListResourceBundle.loadLookup(OpenListResourceBundle.java:146)
at sun.util.resources.OpenListResourceBundle.loadLookupTablesIfNecessary(OpenListResourceBundle.java:128)
at sun.util.resources.OpenListResourceBundle.handleKeySet(OpenListResourceBundle.java:96)
at java.util.ResourceBundle.containsKey(ResourceBundle.java:1807)
at sun.util.locale.provider.LocaleResources.getTimeZoneNames(LocaleResources.java:263)
at sun.util.locale.provider.TimeZoneNameProviderImpl.getDisplayNameArray(TimeZoneNameProviderImpl.java:124)
You can find here a graph of the memory usage of one of the pods :
as you can see, the crash happens at around 18:08.
Here is the namerd config :
admin:
ip: 127.0.0.1
port: 9991
storage:
kind: io.l5d.k8s
experimental: true
namespace: l5d
namers:
- kind: io.l5d.k8s
experimental: true
host: 127.0.0.1
port: 8001
interfaces:
- kind: io.l5d.thriftNameInterpreter
ip: 0.0.0.0
port: 4100
- kind: io.l5d.httpController
ip: 0.0.0.0
port: 4180
Worth noting is that I use kind: io.l5d.k8s, altough i will probably switch to etcd.
Also, here is the output of /admin/metrics.json before the crash (and after the memory exception)
{
"jvm/start_time": 1496853770000,
"jvm/application_time_millis": 2862850,
"jvm/classes/total_loaded": 9053,
"jvm/classes/current_loaded": 8993,
"jvm/classes/total_unloaded": 60,
"jvm/postGC/Par_Survivor_Space/max": 34865152,
"jvm/postGC/Par_Survivor_Space/used": 0,
"jvm/postGC/CMS_Old_Gen/max": 724828160,
"jvm/postGC/CMS_Old_Gen/used": 11649848,
"jvm/postGC/Par_Eden_Space/max": 279183360,
"jvm/postGC/Par_Eden_Space/used": 0,
"jvm/postGC/used": 11649848,
"jvm/nonheap/committed": 64679936,
"jvm/nonheap/max": -1,
"jvm/nonheap/used": 59097064,
"jvm/tenuring_threshold": 1,
"jvm/thread/daemon_count": 34,
"jvm/thread/count": 35,
"jvm/thread/peak_count": 38,
"jvm/mem/postGC/Par_Survivor_Space/max": 34865152,
"jvm/mem/postGC/Par_Survivor_Space/used": 0,
"jvm/mem/postGC/CMS_Old_Gen/max": 724828160,
"jvm/mem/postGC/CMS_Old_Gen/used": 11649848,
"jvm/mem/postGC/Par_Eden_Space/max": 279183360,
"jvm/mem/postGC/Par_Eden_Space/used": 0,
"jvm/mem/postGC/used": 11649848,
"jvm/mem/metaspace/max_capacity": 1098907650,
"jvm/mem/buffer/direct/max": 546560,
"jvm/mem/buffer/direct/count": 20,
"jvm/mem/buffer/direct/used": 546561,
"jvm/mem/buffer/mapped/max": 0,
"jvm/mem/buffer/mapped/count": 0,
"jvm/mem/buffer/mapped/used": 0,
"jvm/mem/allocations/eden/bytes": 2599131900,
"jvm/mem/current/used": 101241880,
"jvm/mem/current/CMS_Old_Gen/max": 724828160,
"jvm/mem/current/CMS_Old_Gen/used": 11649848,
"jvm/mem/current/Metaspace/max": -1,
"jvm/mem/current/Metaspace/used": 48740048,
"jvm/mem/current/Par_Eden_Space/max": 279183360,
"jvm/mem/current/Par_Eden_Space/used": 30494400,
"jvm/mem/current/Par_Survivor_Space/max": 34865152,
"jvm/mem/current/Par_Survivor_Space/used": 0,
"jvm/mem/current/Compressed_Class_Space/max": 1073741820,
"jvm/mem/current/Compressed_Class_Space/used": 8129296,
"jvm/mem/current/Code_Cache/max": 50331648,
"jvm/mem/current/Code_Cache/used": 2228288,
"jvm/num_cpus": 4,
"jvm/gc/msec": 530681,
"jvm/gc/eden/pause_msec.count": 0,
"jvm/gc/ParNew/msec": 3415,
"jvm/gc/ParNew/cycles": 455,
"jvm/gc/ConcurrentMarkSweep/msec": 527266,
"jvm/gc/ConcurrentMarkSweep/cycles": 302,
"jvm/gc/cycles": 757,
"jvm/fd_limit": 1048576,
"jvm/compilation/time_msec": 3888,
"jvm/uptime": 3395264,
"jvm/safepoint/sync_time_millis": 266,
"jvm/safepoint/total_time_millis": 531750,
"jvm/safepoint/count": 4232,
"jvm/heap/committed": 1038876670,
"jvm/heap/max": 1038876670,
"jvm/heap/used": 42144248,
"jvm/fd_count": 97,
"zk2/inet/dns/queue_size": 0,
"zk2/inet/dns/successes": 0,
"zk2/inet/dns/cache/evicts": 0,
"zk2/inet/dns/cache/size": 0,
"zk2/inet/dns/cache/hit_rate": 1,
"zk2/inet/dns/dns_lookups": 0,
"zk2/inet/dns/failures": 0,
"zk2/inet/dns/dns_lookup_failures": 0,
"zk2/inet/dns/lookup_ms.count": 0,
"zk2/observed_serversets": 0,
"zk2/session_cache_size": 0,
"retries.count": 0,
"retries/budget_exhausted": 0,
"io.l5d.thriftNameInterpreter/sent_bytes": 8851,
"io.l5d.thriftNameInterpreter/connection_received_bytes.count": 0,
"io.l5d.thriftNameInterpreter/connection_duration.count": 0,
"io.l5d.thriftNameInterpreter/connects": 3,
"io.l5d.thriftNameInterpreter/success": 6,
"io.l5d.thriftNameInterpreter/request_latency_ms.count": 0,
"io.l5d.thriftNameInterpreter/thriftmux/connects": 3,
"io.l5d.thriftNameInterpreter/thriftmux/downgraded_connects": 0,
"io.l5d.thriftNameInterpreter/mux/orphaned_tdiscard": 0,
"io.l5d.thriftNameInterpreter/mux/duplicate_tag": 0,
"io.l5d.thriftNameInterpreter/received_bytes": 10969,
"io.l5d.thriftNameInterpreter/read_timeout": 0,
"io.l5d.thriftNameInterpreter/write_timeout": 0,
"io.l5d.thriftNameInterpreter/bind/failures": 0,
"io.l5d.thriftNameInterpreter/bind/success": 4,
"io.l5d.thriftNameInterpreter/bind/requests": 8,
"io.l5d.thriftNameInterpreter/connection_sent_bytes.count": 0,
"io.l5d.thriftNameInterpreter/connection_requests.count": 0,
"io.l5d.thriftNameInterpreter/transit_latency_ms.count": 0,
"io.l5d.thriftNameInterpreter/socket_unwritable_ms": 0,
"io.l5d.thriftNameInterpreter/closes": 0,
"io.l5d.thriftNameInterpreter/addr/failures": 0,
"io.l5d.thriftNameInterpreter/addr/success": 2,
"io.l5d.thriftNameInterpreter/addr/requests": 4,
"io.l5d.thriftNameInterpreter/request_payload_bytes.count": 0,
"io.l5d.thriftNameInterpreter/socket_writable_ms": 0,
"io.l5d.thriftNameInterpreter/response_payload_bytes.count": 0,
"io.l5d.thriftNameInterpreter/dtab/size.count": 0,
"io.l5d.thriftNameInterpreter/requests": 6,
"io.l5d.thriftNameInterpreter/pending": 6,
"io.l5d.thriftNameInterpreter/protocol/thriftmux": 1,
"io.l5d.thriftNameInterpreter/handletime_us.count": 0,
"io.l5d.thriftNameInterpreter/connections": 3,
"io.l5d.httpController/success": 0,
"io.l5d.httpController/request_latency_ms.count": 0,
"io.l5d.httpController/nacks": 0,
"io.l5d.httpController/transit_latency_ms.count": 0,
"io.l5d.httpController/nonretryable_nacks": 0,
"io.l5d.httpController/dtab/size.count": 0,
"io.l5d.httpController/requests": 0,
"io.l5d.httpController/pending": 0,
"io.l5d.httpController/handletime_us.count": 0,
"clnt/zipkin-tracer/service_creation/service_acquisition_latency_ms.count": 0,
"clnt/zipkin-tracer/retries/requeues_per_request.count": 0,
"clnt/zipkin-tracer/retries/request_limit": 0,
"clnt/zipkin-tracer/retries/budget_exhausted": 0,
"clnt/zipkin-tracer/retries/cannot_retry": 0,
"clnt/zipkin-tracer/retries/not_open": 0,
"clnt/zipkin-tracer/retries/budget": 100,
"clnt/zipkin-tracer/retries/requeues": 0,
"clnt/zipkin-tracer/nack_admission_control/dropped_requests": 0,
"clnt/zipkin-tracer/tries/request_latency_ms.count": 0,
"clnt/zipkin-tracer/tries/success": 0,
"clnt/zipkin-tracer/tries/pending": 0,
"clnt/zipkin-tracer/tries/requests": 0,
"clnt/zipkin-tracer/loadbalancer/size": 1,
"clnt/zipkin-tracer/loadbalancer/rebuilds": 1,
"clnt/zipkin-tracer/loadbalancer/closed": 0,
"clnt/zipkin-tracer/loadbalancer/load": 0,
"clnt/zipkin-tracer/loadbalancer/meanweight": 1,
"clnt/zipkin-tracer/loadbalancer/adds": 1,
"clnt/zipkin-tracer/loadbalancer/updates": 1,
"clnt/zipkin-tracer/loadbalancer/algorithm/p2c_least_loaded": 1,
"clnt/zipkin-tracer/loadbalancer/available": 1,
"clnt/zipkin-tracer/loadbalancer/max_effort_exhausted": 0,
"clnt/zipkin-tracer/loadbalancer/busy": 0,
"clnt/zipkin-tracer/loadbalancer/removes": 0,
"clnt//$/inet/localhost/8001/connect_latency_ms.count": 0,
"clnt//$/inet/localhost/8001/failed_connect_latency_ms.count": 0,
"clnt//$/inet/localhost/8001/sent_bytes": 1012,
"clnt//$/inet/localhost/8001/service_creation/service_acquisition_latency_ms.count": 0,
"clnt//$/inet/localhost/8001/connection_received_bytes.count": 0,
"clnt//$/inet/localhost/8001/connection_duration.count": 0,
"clnt//$/inet/localhost/8001/failure_accrual/removals": 0,
"clnt//$/inet/localhost/8001/failure_accrual/probes": 0,
"clnt//$/inet/localhost/8001/failure_accrual/removed_for_ms": 0,
"clnt//$/inet/localhost/8001/failure_accrual/revivals": 0,
"clnt//$/inet/localhost/8001/connects": 2,
"clnt//$/inet/localhost/8001/pool_num_waited": 0,
"clnt//$/inet/localhost/8001/success": 4,
"clnt//$/inet/localhost/8001/request_latency_ms.count": 0,
"clnt//$/inet/localhost/8001/pool_waiters": 0,
"clnt//$/inet/localhost/8001/retries/requeues_per_request.count": 0,
"clnt//$/inet/localhost/8001/retries/request_limit": 0,
"clnt//$/inet/localhost/8001/retries/budget_exhausted": 0,
"clnt//$/inet/localhost/8001/retries/cannot_retry": 0,
"clnt//$/inet/localhost/8001/retries/not_open": 0,
"clnt//$/inet/localhost/8001/retries/budget": 200,
"clnt//$/inet/localhost/8001/retries/requeues": 0,
"clnt//$/inet/localhost/8001/received_bytes": 10601,
"clnt//$/inet/localhost/8001/namer/nametreecache/expires": 0,
"clnt//$/inet/localhost/8001/namer/nametreecache/evicts": 0,
"clnt//$/inet/localhost/8001/namer/nametreecache/misses": 2,
"clnt//$/inet/localhost/8001/namer/nametreecache/oneshots": 0,
"clnt//$/inet/localhost/8001/namer/dtabcache/expires": 0,
"clnt//$/inet/localhost/8001/namer/dtabcache/evicts": 0,
"clnt//$/inet/localhost/8001/namer/dtabcache/misses": 2,
"clnt//$/inet/localhost/8001/namer/dtabcache/oneshots": 0,
"clnt//$/inet/localhost/8001/namer/namecache/expires": 0,
"clnt//$/inet/localhost/8001/namer/namecache/evicts": 0,
"clnt//$/inet/localhost/8001/namer/namecache/misses": 2,
"clnt//$/inet/localhost/8001/namer/namecache/oneshots": 0,
"clnt//$/inet/localhost/8001/namer/bind_latency_us.count": 0,
"clnt//$/inet/localhost/8001/failfast/marked_available": 0,
"clnt//$/inet/localhost/8001/failfast/unhealthy_num_tries": 0,
"clnt//$/inet/localhost/8001/failfast/marked_dead": 0,
"clnt//$/inet/localhost/8001/failfast/unhealthy_for_ms": 0,
"clnt//$/inet/localhost/8001/connection_sent_bytes.count": 0,
"clnt//$/inet/localhost/8001/connection_requests.count": 0,
"clnt//$/inet/localhost/8001/pool_num_too_many_waiters": 0,
"clnt//$/inet/localhost/8001/socket_unwritable_ms": 0,
"clnt//$/inet/localhost/8001/closes": 0,
"clnt//$/inet/localhost/8001/pool_cached": 1,
"clnt//$/inet/localhost/8001/nack_admission_control/dropped_requests": 0,
"clnt//$/inet/localhost/8001/pool_size": 1,
"clnt//$/inet/localhost/8001/available": 2,
"clnt//$/inet/localhost/8001/socket_writable_ms": 0,
"clnt//$/inet/localhost/8001/cancelled_connects": 0,
"clnt//$/inet/localhost/8001/dtab/size.count": 0,
"clnt//$/inet/localhost/8001/requests": 4,
"clnt//$/inet/localhost/8001/loadbalancer/size": 2,
"clnt//$/inet/localhost/8001/loadbalancer/rebuilds": 1190,
"clnt//$/inet/localhost/8001/loadbalancer/closed": 0,
"clnt//$/inet/localhost/8001/loadbalancer/load": 1,
"clnt//$/inet/localhost/8001/loadbalancer/meanweight": 2,
"clnt//$/inet/localhost/8001/loadbalancer/adds": 2,
"clnt//$/inet/localhost/8001/loadbalancer/updates": 1190,
"clnt//$/inet/localhost/8001/loadbalancer/algorithm/p2c_least_loaded": 2,
"clnt//$/inet/localhost/8001/loadbalancer/available": 2,
"clnt//$/inet/localhost/8001/loadbalancer/max_effort_exhausted": 0,
"clnt//$/inet/localhost/8001/loadbalancer/busy": 0,
"clnt//$/inet/localhost/8001/loadbalancer/removes": 0,
"clnt//$/inet/localhost/8001/pending": 0,
"clnt//$/inet/localhost/8001/connections": 2,
"clnt/dispatcher/serial/queue_size": 0,
"delegationcache/inactive": 0,
"delegationcache/active": 0,
"buffer/resetCount": 0,
"larger_than_threadlocal_out_buffer": 0,
"namer/#/io.l5d.k8s/ppd/sent_bytes": 970,
"namer/#/io.l5d.k8s/ppd/failure_accrual/removals": 0,
"namer/#/io.l5d.k8s/ppd/failure_accrual/probes": 0,
"namer/#/io.l5d.k8s/ppd/failure_accrual/removed_for_ms": 0,
"namer/#/io.l5d.k8s/ppd/failure_accrual/revivals": 0,
"namer/#/io.l5d.k8s/ppd/connects": 2,
"namer/#/io.l5d.k8s/ppd/pool_num_waited": 0,
"namer/#/io.l5d.k8s/ppd/success": 4,
"namer/#/io.l5d.k8s/ppd/pool_waiters": 0,
"namer/#/io.l5d.k8s/ppd/retries/request_limit": 0,
"namer/#/io.l5d.k8s/ppd/retries/budget_exhausted": 0,
"namer/#/io.l5d.k8s/ppd/retries/cannot_retry": 0,
"namer/#/io.l5d.k8s/ppd/retries/not_open": 0,
"namer/#/io.l5d.k8s/ppd/retries/budget": 200,
"namer/#/io.l5d.k8s/ppd/retries/requeues": 0,
"namer/#/io.l5d.k8s/ppd/received_bytes": 12691,
"namer/#/io.l5d.k8s/ppd/namer/nametreecache/expires": 0,
"namer/#/io.l5d.k8s/ppd/namer/nametreecache/evicts": 0,
"namer/#/io.l5d.k8s/ppd/namer/nametreecache/misses": 2,
"namer/#/io.l5d.k8s/ppd/namer/nametreecache/oneshots": 0,
"namer/#/io.l5d.k8s/ppd/namer/dtabcache/expires": 0,
"namer/#/io.l5d.k8s/ppd/namer/dtabcache/evicts": 0,
"namer/#/io.l5d.k8s/ppd/namer/dtabcache/misses": 2,
"namer/#/io.l5d.k8s/ppd/namer/dtabcache/oneshots": 0,
"namer/#/io.l5d.k8s/ppd/namer/namecache/expires": 0,
"namer/#/io.l5d.k8s/ppd/namer/namecache/evicts": 0,
"namer/#/io.l5d.k8s/ppd/namer/namecache/misses": 2,
"namer/#/io.l5d.k8s/ppd/namer/namecache/oneshots": 0,
"namer/#/io.l5d.k8s/ppd/failfast/marked_available": 0,
"namer/#/io.l5d.k8s/ppd/failfast/unhealthy_num_tries": 0,
"namer/#/io.l5d.k8s/ppd/failfast/marked_dead": 0,
"namer/#/io.l5d.k8s/ppd/failfast/unhealthy_for_ms": 0,
"namer/#/io.l5d.k8s/ppd/pool_num_too_many_waiters": 0,
"namer/#/io.l5d.k8s/ppd/socket_unwritable_ms": 0,
"namer/#/io.l5d.k8s/ppd/closes": 0,
"namer/#/io.l5d.k8s/ppd/pool_cached": 0,
"namer/#/io.l5d.k8s/ppd/nack_admission_control/dropped_requests": 0,
"namer/#/io.l5d.k8s/ppd/pool_size": 2,
"namer/#/io.l5d.k8s/ppd/available": 2,
"namer/#/io.l5d.k8s/ppd/socket_writable_ms": 0,
"namer/#/io.l5d.k8s/ppd/cancelled_connects": 0,
"namer/#/io.l5d.k8s/ppd/requests": 4,
"namer/#/io.l5d.k8s/ppd/loadbalancer/size": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/rebuilds": 48,
"namer/#/io.l5d.k8s/ppd/loadbalancer/closed": 0,
"namer/#/io.l5d.k8s/ppd/loadbalancer/load": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/meanweight": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/adds": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/updates": 48,
"namer/#/io.l5d.k8s/ppd/loadbalancer/algorithm/p2c_least_loaded": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/available": 2,
"namer/#/io.l5d.k8s/ppd/loadbalancer/max_effort_exhausted": 0,
"namer/#/io.l5d.k8s/ppd/loadbalancer/busy": 0,
"namer/#/io.l5d.k8s/ppd/loadbalancer/removes": 0,
"namer/#/io.l5d.k8s/ppd/pending": 0,
"namer/#/io.l5d.k8s/ppd/connections": 2,
"finagle/future_pool/active_tasks": 0,
"finagle/future_pool/pool_size": 8,
"finagle/future_pool/completed_tasks": 7334,
"finagle/netty4/reference_leaks": 0,
"finagle/aperture_coordinate": -2,
"addrcache/inactive": 0,
"addrcache/active": 2,
"scheduler/dispatches": 8930,
"scheduler/blocking_ms": 0,
"inet/dns/queue_size": 0,
"inet/dns/successes": 4870,
"inet/dns/cache/evicts": 0,
"inet/dns/cache/size": 0,
"inet/dns/cache/hit_rate": 1,
"inet/dns/dns_lookups": 2476,
"inet/dns/failures": 0,
"inet/dns/dns_lookup_failures": 0,
"inet/dns/lookup_ms.count": 64,
"inet/dns/lookup_ms.max": 6,
"inet/dns/lookup_ms.min": 0,
"inet/dns/lookup_ms.p50": 0,
"inet/dns/lookup_ms.p90": 1,
"inet/dns/lookup_ms.p95": 2,
"inet/dns/lookup_ms.p99": 3,
"inet/dns/lookup_ms.p9990": 6,
"inet/dns/lookup_ms.p9999": 6,
"inet/dns/lookup_ms.sum": 25,
"inet/dns/lookup_ms.avg": 0.390625,
"bindingcache/inactive": 0,
"bindingcache/active": 4,
"toggles/com.twitter.finagle.mux/checksum": 1900976900,
"toggles/com.twitter.finagle.http/checksum": 1471331970,
"toggles/com.twitter.finagle.netty4/checksum": 2835989500,
"toggles/com.twitter.finagle.core/checksum": 239388080,
"zipkin/zipkin/log_span/ok": 0,
"zipkin/zipkin/log_span/try_later": 0,
"dispatcher/serial/queue_size": 0,
"namer/#/io.l5d.k8s/kube-system/connect_latency_ms.count": 0,
"namer/#/io.l5d.k8s/kube-system/failed_connect_latency_ms.count": 0,
"namer/#/io.l5d.k8s/kube-system/sent_bytes": 1502,
"namer/#/io.l5d.k8s/kube-system/service_creation/service_acquisition_latency_ms.count": 0,
"namer/#/io.l5d.k8s/kube-system/connection_received_bytes.count": 0,
"namer/#/io.l5d.k8s/kube-system/connection_duration.count": 0,
"namer/#/io.l5d.k8s/kube-system/failure_accrual/removals": 0,
"namer/#/io.l5d.k8s/kube-system/failure_accrual/probes": 0,
"namer/#/io.l5d.k8s/kube-system/failure_accrual/removed_for_ms": 0,
"namer/#/io.l5d.k8s/kube-system/failure_accrual/revivals": 0,
"namer/#/io.l5d.k8s/kube-system/connects": 2,
"namer/#/io.l5d.k8s/kube-system/pool_num_waited": 0,
"namer/#/io.l5d.k8s/kube-system/success": 6,
"namer/#/io.l5d.k8s/kube-system/request_latency_ms.count": 0,
"namer/#/io.l5d.k8s/kube-system/pool_waiters": 0,
"namer/#/io.l5d.k8s/kube-system/retries/requeues_per_request.count": 0,
"namer/#/io.l5d.k8s/kube-system/retries/request_limit": 0,
"namer/#/io.l5d.k8s/kube-system/retries/budget_exhausted": 0,
"namer/#/io.l5d.k8s/kube-system/retries/cannot_retry": 0,
"namer/#/io.l5d.k8s/kube-system/retries/not_open": 0,
"namer/#/io.l5d.k8s/kube-system/retries/budget": 200,
"namer/#/io.l5d.k8s/kube-system/retries/requeues": 0,
"namer/#/io.l5d.k8s/kube-system/received_bytes": 1880905,
"namer/#/io.l5d.k8s/kube-system/namer/nametreecache/expires": 0,
"namer/#/io.l5d.k8s/kube-system/namer/nametreecache/evicts": 0,
"namer/#/io.l5d.k8s/kube-system/namer/nametreecache/misses": 2,
"namer/#/io.l5d.k8s/kube-system/namer/nametreecache/oneshots": 0,
"namer/#/io.l5d.k8s/kube-system/namer/dtabcache/expires": 0,
"namer/#/io.l5d.k8s/kube-system/namer/dtabcache/evicts": 0,
"namer/#/io.l5d.k8s/kube-system/namer/dtabcache/misses": 2,
"namer/#/io.l5d.k8s/kube-system/namer/dtabcache/oneshots": 0,
"namer/#/io.l5d.k8s/kube-system/namer/namecache/expires": 0,
"namer/#/io.l5d.k8s/kube-system/namer/namecache/evicts": 0,
"namer/#/io.l5d.k8s/kube-system/namer/namecache/misses": 2,
"namer/#/io.l5d.k8s/kube-system/namer/namecache/oneshots": 0,
"namer/#/io.l5d.k8s/kube-system/namer/bind_latency_us.count": 0,
"namer/#/io.l5d.k8s/kube-system/failfast/marked_available": 0,
"namer/#/io.l5d.k8s/kube-system/failfast/unhealthy_num_tries": 0,
"namer/#/io.l5d.k8s/kube-system/failfast/marked_dead": 0,
"namer/#/io.l5d.k8s/kube-system/failfast/unhealthy_for_ms": 0,
"namer/#/io.l5d.k8s/kube-system/connection_sent_bytes.count": 0,
"namer/#/io.l5d.k8s/kube-system/connection_requests.count": 0,
"namer/#/io.l5d.k8s/kube-system/pool_num_too_many_waiters": 0,
"namer/#/io.l5d.k8s/kube-system/socket_unwritable_ms": 0,
"namer/#/io.l5d.k8s/kube-system/closes": 1,
"namer/#/io.l5d.k8s/kube-system/pool_cached": 0,
"namer/#/io.l5d.k8s/kube-system/nack_admission_control/dropped_requests": 0,
"namer/#/io.l5d.k8s/kube-system/pool_size": 1,
"namer/#/io.l5d.k8s/kube-system/exn/java.lang.OutOfMemoryError": 1,
"namer/#/io.l5d.k8s/kube-system/available": 2,
"namer/#/io.l5d.k8s/kube-system/socket_writable_ms": 0,
"namer/#/io.l5d.k8s/kube-system/cancelled_connects": 0,
"namer/#/io.l5d.k8s/kube-system/dtab/size.count": 0,
"namer/#/io.l5d.k8s/kube-system/requests": 6,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/size": 2,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/rebuilds": 1190,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/closed": 0,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/load": 1,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/meanweight": 2,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/adds": 2,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/updates": 1190,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/algorithm/p2c_least_loaded": 2,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/available": 2,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/max_effort_exhausted": 0,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/busy": 0,
"namer/#/io.l5d.k8s/kube-system/loadbalancer/removes": 0,
"namer/#/io.l5d.k8s/kube-system/pending": 0,
"namer/#/io.l5d.k8s/kube-system/connections": 1
}