decouple workqueue metrics from prometheus

Chao Xu · Chao Xu · commit b094d6c49ce7 · 2016-09-30T15:10:02.000-07:00
diff --git a/federation/pkg/federation-controller/replicaset/replicasetcontroller.go b/federation/pkg/federation-controller/replicaset/replicasetcontroller.go
@@ -43,6 +43,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/flowcontrol"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/federation/pkg/federation-controller/service/cluster_helper.go b/federation/pkg/federation-controller/service/cluster_helper.go
@@ -28,6 +28,7 @@ import (
 	pkg_runtime "k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"reflect"
diff --git a/federation/pkg/federation-controller/service/servicecontroller.go b/federation/pkg/federation-controller/service/servicecontroller.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"k8s.io/kubernetes/pkg/conversion"
diff --git a/pkg/controller/certificates/controller.go b/pkg/controller/certificates/controller.go
@@ -34,6 +34,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/cloudflare/cfssl/config"
diff --git a/pkg/controller/daemon/daemoncontroller.go b/pkg/controller/daemon/daemoncontroller.go
@@ -39,6 +39,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
 
diff --git a/pkg/controller/deployment/deployment_controller.go b/pkg/controller/deployment/deployment_controller.go
@@ -42,6 +42,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/disruption/disruption.go b/pkg/controller/disruption/disruption.go
@@ -36,6 +36,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go
@@ -42,6 +42,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
diff --git a/pkg/controller/garbagecollector/garbagecollector.go b/pkg/controller/garbagecollector/garbagecollector.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/garbagecollector/garbagecollector_test.go b/pkg/controller/garbagecollector/garbagecollector_test.go
@@ -41,6 +41,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/json"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 )
 
 func TestNewGarbageCollector(t *testing.T) {
diff --git a/pkg/controller/job/jobcontroller.go b/pkg/controller/job/jobcontroller.go
@@ -38,6 +38,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go
@@ -30,6 +30,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
diff --git a/pkg/controller/petset/pet_set.go b/pkg/controller/petset/pet_set.go
@@ -36,6 +36,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
diff --git a/pkg/controller/replicaset/replica_set.go b/pkg/controller/replicaset/replica_set.go
@@ -44,6 +44,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/replication/replication_controller.go b/pkg/controller/replication/replication_controller.go
@@ -43,6 +43,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/resourcequota/resource_quota_controller.go b/pkg/controller/resourcequota/resource_quota_controller.go
@@ -32,6 +32,7 @@ import (
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/service/servicecontroller.go b/pkg/controller/service/servicecontroller.go
@@ -39,6 +39,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/controller/serviceaccount/tokens_controller.go b/pkg/controller/serviceaccount/tokens_controller.go
@@ -38,6 +38,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/util/workqueue"
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	"k8s.io/kubernetes/pkg/watch"
 )
 
diff --git a/pkg/util/workqueue/metrics.go b/pkg/util/workqueue/metrics.go
@@ -17,63 +17,54 @@ limitations under the License.
 package workqueue
 
 import (
+	"sync"
 	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
+// This file provides abstractions for setting the provider (e.g., prometheus)
+// of metrics.
+
 type queueMetrics interface {
 	add(item t)
 	get(item t)
 	done(item t)
 }
 
-type defaultQueueMetrics struct {
-	depth                prometheus.Gauge
-	adds                 prometheus.Counter
-	latency              prometheus.Summary
-	workDuration         prometheus.Summary
-	addTimes             map[t]time.Time
-	processingStartTimes map[t]time.Time
+// GaugeMetric represents a single numerical value that can arbitrarily go up
+// and down.
+type GaugeMetric interface {
+	Inc()
+	Dec()
 }
 
-func newQueueMetrics(name string) queueMetrics {
-	var ret *defaultQueueMetrics
-	if len(name) == 0 {
-		return ret
-	}
+// CounterMetric represents a single numerical value that only ever
+// goes up.
+type CounterMetric interface {
+	Inc()
+}
 
-	ret = &defaultQueueMetrics{
-		depth: prometheus.NewGauge(prometheus.GaugeOpts{
-			Subsystem: name,
-			Name:      "depth",
-			Help:      "Current depth of workqueue: " + name,
-		}),
-		adds: prometheus.NewCounter(prometheus.CounterOpts{
-			Subsystem: name,
-			Name:      "adds",
-			Help:      "Total number of adds handled by workqueue: " + name,
-		}),
-		latency: prometheus.NewSummary(prometheus.SummaryOpts{
-			Subsystem: name,
-			Name:      "queue_latency",
-			Help:      "How long an item stays in workqueue" + name + " before being requested.",
-		}),
-		workDuration: prometheus.NewSummary(prometheus.SummaryOpts{
-			Subsystem: name,
-			Name:      "work_duration",
-			Help:      "How long processing an item from workqueue" + name + " takes.",
-		}),
-		addTimes:             map[t]time.Time{},
-		processingStartTimes: map[t]time.Time{},
-	}
+// SummaryMetric captures individual observations.
+type SummaryMetric interface {
+	Observe(float64)
+}
 
-	prometheus.Register(ret.depth)
-	prometheus.Register(ret.adds)
-	prometheus.Register(ret.latency)
-	prometheus.Register(ret.workDuration)
+type noopMetric struct{}
 
-	return ret
+func (noopMetric) Inc()            {}
+func (noopMetric) Dec()            {}
+func (noopMetric) Observe(float64) {}
+
+type defaultQueueMetrics struct {
+	// current depth of a workqueue
+	depth GaugeMetric
+	// total number of adds handled by a workqueue
+	adds CounterMetric
+	// how long an item stays in a workqueue
+	latency SummaryMetric
+	// how long processing an item from a workqueue takes
+	workDuration         SummaryMetric
+	addTimes             map[t]time.Time
+	processingStartTimes map[t]time.Time
 }
 
 func (m *defaultQueueMetrics) add(item t) {
@@ -122,32 +113,81 @@ type retryMetrics interface {
 }
 
 type defaultRetryMetrics struct {
-	retries prometheus.Counter
+	retries CounterMetric
 }
 
-func newRetryMetrics(name string) retryMetrics {
-	var ret *defaultRetryMetrics
-	if len(name) == 0 {
-		return ret
+func (m *defaultRetryMetrics) retry() {
+	if m == nil {
+		return
 	}
 
-	ret = &defaultRetryMetrics{
-		retries: prometheus.NewCounter(prometheus.CounterOpts{
-			Subsystem: name,
-			Name:      "retries",
-			Help:      "Total number of retries handled by workqueue: " + name,
-		}),
-	}
+	m.retries.Inc()
+}
+
+// MetricsProvider generates various metrics used by the queue.
+type MetricsProvider interface {
+	NewDepthMetric(name string) GaugeMetric
+	NewAddsMetric(name string) CounterMetric
+	NewLatencyMetric(name string) SummaryMetric
+	NewWorkDurationMetric(name string) SummaryMetric
+	NewRetriesMetric(name string) CounterMetric
+}
 
-	prometheus.Register(ret.retries)
+type noopMetricsProvider struct{}
 
-	return ret
+func (_ noopMetricsProvider) NewDepthMetric(name string) GaugeMetric {
+	return noopMetric{}
 }
 
-func (m *defaultRetryMetrics) retry() {
-	if m == nil {
-		return
+func (_ noopMetricsProvider) NewAddsMetric(name string) CounterMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewLatencyMetric(name string) SummaryMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewWorkDurationMetric(name string) SummaryMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewRetriesMetric(name string) CounterMetric {
+	return noopMetric{}
+}
+
+var metricsFactory = struct {
+	metricsProvider MetricsProvider
+	setProviders    sync.Once
+}{}
+
+func newQueueMetrics(name string) queueMetrics {
+	var ret *defaultQueueMetrics
+	if len(name) == 0 {
+		return ret
+	}
+	return &defaultQueueMetrics{
+		depth:                metricsFactory.metricsProvider.NewDepthMetric(name),
+		adds:                 metricsFactory.metricsProvider.NewAddsMetric(name),
+		latency:              metricsFactory.metricsProvider.NewLatencyMetric(name),
+		workDuration:         metricsFactory.metricsProvider.NewWorkDurationMetric(name),
+		addTimes:             map[t]time.Time{},
+		processingStartTimes: map[t]time.Time{},
 	}
+}
 
-	m.retries.Inc()
+func newRetryMetrics(name string) retryMetrics {
+	var ret *defaultRetryMetrics
+	if len(name) == 0 {
+		return ret
+	}
+	return &defaultRetryMetrics{
+		retries: metricsFactory.metricsProvider.NewRetriesMetric(name),
+	}
+}
+
+// SetProvider sets the metrics provider of the metricsFactory.
+func SetProvider(metricsProvider MetricsProvider) {
+	metricsFactory.setProviders.Do(func() {
+		metricsFactory.metricsProvider = metricsProvider
+	})
 }
diff --git a/pkg/util/workqueue/prometheus/prometheus.go b/pkg/util/workqueue/prometheus/prometheus.go
diff --git a/plugin/pkg/admission/resourcequota/controller.go b/plugin/pkg/admission/resourcequota/controller.go
diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go
diff --git a/plugin/pkg/scheduler/algorithm/priorities/interpod_affinity.go b/plugin/pkg/scheduler/algorithm/priorities/interpod_affinity.go
diff --git a/plugin/pkg/scheduler/algorithm/priorities/selector_spreading.go b/plugin/pkg/scheduler/algorithm/priorities/selector_spreading.go
diff --git a/plugin/pkg/scheduler/generic_scheduler.go b/plugin/pkg/scheduler/generic_scheduler.go

Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,7 @@ import (`
`43`	`43`	`"k8s.io/kubernetes/pkg/util/flowcontrol"`
`44`	`44`	`"k8s.io/kubernetes/pkg/util/wait"`
`45`	`45`	`"k8s.io/kubernetes/pkg/util/workqueue"`
	`46`	`+ _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"`
`46`	`47`	`"k8s.io/kubernetes/pkg/watch"`
`47`	`48`	`)`
`48`	`49`
Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@ import (`
`42`	`42`	`utilruntime "k8s.io/kubernetes/pkg/util/runtime"`
`43`	`43`	`"k8s.io/kubernetes/pkg/util/wait"`
`44`	`44`	`"k8s.io/kubernetes/pkg/util/workqueue"`
	`45`	`+ _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"`
`45`	`46`	`"k8s.io/kubernetes/pkg/watch"`
`46`	`47`	`)`
`47`	`48`
Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,7 @@ import (`
`40`	`40`	`"k8s.io/kubernetes/pkg/util/sets"`
`41`	`41`	`"k8s.io/kubernetes/pkg/util/wait"`
`42`	`42`	`"k8s.io/kubernetes/pkg/util/workqueue"`
	`43`	`+ _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"`
`43`	`44`	`"k8s.io/kubernetes/pkg/watch"`
`44`	`45`	`)`
`45`	`46`
Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,7 @@ import (`
`41`	`41`	`"k8s.io/kubernetes/pkg/util/json"`
`42`	`42`	`"k8s.io/kubernetes/pkg/util/sets"`
`43`	`43`	`"k8s.io/kubernetes/pkg/util/workqueue"`
	`44`	`+ _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"`
`44`	`45`	`)`
`45`	`46`
`46`	`47`	`func TestNewGarbageCollector(t *testing.T) {`