Add gauge metric for master of leader election.

Fixes #71730
0 indicates standby, 1 indicates master, label indicates which lease.
Tweaked name and documentation
Factored in Mike Danese feedback.
Removed dependency on prometheus from client-go using adapter.
Centralized adapter import.
Fixed godeps
Fixed boilerplate.
Put in fixes for caesarxuchao

cmd/cloud-controller-manager/BUILD

@@ -20,8 +20,8 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
     deps = [
         "//cmd/cloud-controller-manager/app:go_default_library",
-        "//pkg/client/metrics/prometheus:go_default_library",
         "//pkg/cloudprovider/providers:go_default_library",
+        "//pkg/util/prometheusclientgo:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

cmd/cloud-controller-manager/controller-manager.go

@@ -27,11 +27,11 @@ import (
 
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
-	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
 	// NOTE: Importing all in-tree cloud-providers is not required when
 	// implementing an out-of-tree cloud-provider.
 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
-	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
+	_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
+	_ "k8s.io/kubernetes/pkg/version/prometheus"      // for version metric registration
 )
 
 func main() {

cmd/kube-apiserver/BUILD

@@ -20,7 +20,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",
-        "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/util/prometheusclientgo:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

cmd/kube-apiserver/apiserver.go

@@ -27,8 +27,8 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
-	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
-	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"      // for version metric registration
 )
 
 func main() {

cmd/kube-controller-manager/BUILD

@@ -20,9 +20,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager",
     deps = [
         "//cmd/kube-controller-manager/app:go_default_library",
-        "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/util/prometheusclientgo:go_default_library",
-        "//pkg/util/reflector/prometheus:go_default_library",
-        "//pkg/util/workqueue/prometheus:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

cmd/kube-controller-manager/controller-manager.go

@@ -28,10 +28,8 @@ import (
 
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app"
-	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugin
-	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"      // for version metric registration
-	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
-	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
 )
 
 func main() {

cmd/kube-scheduler/BUILD

@@ -20,7 +20,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
     deps = [
         "//cmd/kube-scheduler/app:go_default_library",
-        "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/util/prometheusclientgo:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

cmd/kube-scheduler/scheduler.go

@@ -27,8 +27,8 @@ import (
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-scheduler/app"
-	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
-	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"      // for version metric registration
 )
 
 func main() {

pkg/controller/garbagecollector/BUILD

@@ -22,8 +22,6 @@ go_library(
     deps = [
         "//pkg/controller:go_default_library",
         "//pkg/controller/garbagecollector/metaonly:go_default_library",
-        "//pkg/util/reflector/prometheus:go_default_library",
-        "//pkg/util/workqueue/prometheus:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/controller/garbagecollector/garbagecollector.go

@@ -39,9 +39,6 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/kubernetes/pkg/controller"
-	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
-	// install the prometheus plugin
-	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
 	// import known versions
 	_ "k8s.io/client-go/kubernetes"
 )

pkg/util/BUILD

@@ -47,6 +47,7 @@ filegroup(
         "//pkg/util/parsers:all-srcs",
         "//pkg/util/pod:all-srcs",
         "//pkg/util/procfs:all-srcs",
+        "//pkg/util/prometheusclientgo:all-srcs",
         "//pkg/util/reflector/prometheus:all-srcs",
         "//pkg/util/removeall:all-srcs",
         "//pkg/util/resizefs:all-srcs",

pkg/util/prometheusclientgo/BUILD

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["adapters.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/util/prometheusclientgo/leaderelection:go_default_library",
+        "//pkg/util/reflector/prometheus:go_default_library",
+        "//pkg/util/workqueue/prometheus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/util/prometheusclientgo/leaderelection:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/util/prometheusclientgo/adapters.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package prometheusclientgo
+
+// Provided metrics needing adapting
+import (
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus"              // for client metric registration
+	_ "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection" // for leader election metric registration
+	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus"              // for reflector metric registration
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"              // for workqueue metric registration
+)

pkg/util/prometheusclientgo/leaderelection/BUILD

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["adapter.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/util/prometheusclientgo/leaderelection/adapter.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/client-go/tools/leaderelection"
+)
+
+// Package prometheus sets the workqueue DefaultMetricsFactory to produce
+// prometheus metrics. To use this package, you just have to import it.
+
+func init() {
+	leaderelection.SetProvider(prometheusMetricsProvider{})
+}
+
+type prometheusMetricsProvider struct{}
+
+func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric {
+	leaderGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "leader_election_master_status",
+			Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
+		},
+		[]string{"name"},
+	)
+	prometheus.Register(leaderGauge)
+	return &switchAdapter{gauge: leaderGauge}
+}
+
+type switchAdapter struct {
+	gauge *prometheus.GaugeVec
+}
+
+func (s *switchAdapter) On(name string) {
+	s.gauge.WithLabelValues(name).Set(1.0)
+}
+
+func (s *switchAdapter) Off(name string) {
+	s.gauge.WithLabelValues(name).Set(0.0)
+}

plugin/pkg/admission/resourcequota/BUILD

@@ -20,8 +20,6 @@ go_library(
         "//pkg/kubeapiserver/admission:go_default_library",
         "//pkg/quota/v1:go_default_library",
         "//pkg/quota/v1/generic:go_default_library",
-        "//pkg/util/reflector/prometheus:go_default_library",
-        "//pkg/util/workqueue/prometheus:go_default_library",
         "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library",
         "//plugin/pkg/admission/resourcequota/apis/resourcequota/install:go_default_library",
         "//plugin/pkg/admission/resourcequota/apis/resourcequota/v1beta1:go_default_library",

plugin/pkg/admission/resourcequota/controller.go

@@ -37,8 +37,6 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	quota "k8s.io/kubernetes/pkg/quota/v1"
 	"k8s.io/kubernetes/pkg/quota/v1/generic"
-	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
-	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
 	resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota"
 )
 

staging/src/k8s.io/client-go/tools/leaderelection/BUILD

@@ -11,6 +11,7 @@ go_library(
     srcs = [
         "healthzadaptor.go",
         "leaderelection.go",
+        "metrics.go",
     ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection",
     importpath = "k8s.io/client-go/tools/leaderelection",

staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go

@@ -89,10 +89,13 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
 	if lec.Lock == nil {
 		return nil, fmt.Errorf("Lock must not be nil.")
 	}
-	return &LeaderElector{
+	le := LeaderElector{
-		config: lec,
+		config:  lec,
-		clock:  clock.RealClock{},
+		clock:   clock.RealClock{},
-	}, nil
+		metrics: globalMetricsFactory.newLeaderMetrics(),
+	}
+	le.metrics.leaderOff(le.config.Name)
+	return &le, nil
 }
 
 type LeaderElectionConfig struct {
@@ -152,6 +155,8 @@ type LeaderElector struct {
 	// clock is wrapper around time to allow for less flaky testing
 	clock clock.Clock
 
+	metrics leaderMetricsAdapter
+
 	// name is the name of the resource lock for debugging
 	name string
 }
@@ -211,6 +216,7 @@ func (le *LeaderElector) acquire(ctx context.Context) bool {
 			return
 		}
 		le.config.Lock.RecordEvent("became leader")
+		le.metrics.leaderOn(le.config.Name)
 		klog.Infof("successfully acquired lease %v", desc)
 		cancel()
 	}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
@@ -246,6 +252,7 @@ func (le *LeaderElector) renew(ctx context.Context) {
 			return
 		}
 		le.config.Lock.RecordEvent("stopped leading")
+		le.metrics.leaderOff(le.config.Name)
 		klog.Infof("failed to renew lease %v: %v", desc, err)
 		cancel()
 	}, le.config.RetryPeriod, ctx.Done())

staging/src/k8s.io/client-go/tools/leaderelection/metrics.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaderelection
+
+import (
+	"sync"
+)
+
+// This file provides abstractions for setting the provider (e.g., prometheus)
+// of metrics.
+
+type leaderMetricsAdapter interface {
+	leaderOn(name string)
+	leaderOff(name string)
+}
+
+// GaugeMetric represents a single numerical value that can arbitrarily go up
+// and down.
+type SwitchMetric interface {
+	On(name string)
+	Off(name string)
+}
+
+type noopMetric struct{}
+
+func (noopMetric) On(name string)  {}
+func (noopMetric) Off(name string) {}
+
+// defaultLeaderMetrics expects the caller to lock before setting any metrics.
+type defaultLeaderMetrics struct {
+	// leader's value indicates if the current process is the owner of name lease
+	leader SwitchMetric
+}
+
+func (m *defaultLeaderMetrics) leaderOn(name string) {
+	if m == nil {
+		return
+	}
+	m.leader.On(name)
+}
+
+func (m *defaultLeaderMetrics) leaderOff(name string) {
+	if m == nil {
+		return
+	}
+	m.leader.Off(name)
+}
+
+type noMetrics struct{}
+
+func (noMetrics) leaderOn(name string)  {}
+func (noMetrics) leaderOff(name string) {}
+
+// MetricsProvider generates various metrics used by the leader election.
+type MetricsProvider interface {
+	NewLeaderMetric() SwitchMetric
+}
+
+type noopMetricsProvider struct{}
+
+func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric {
+	return noopMetric{}
+}
+
+var globalMetricsFactory = leaderMetricsFactory{
+	metricsProvider: noopMetricsProvider{},
+}
+
+type leaderMetricsFactory struct {
+	metricsProvider MetricsProvider
+
+	onlyOnce sync.Once
+}
+
+func (f *leaderMetricsFactory) setProvider(mp MetricsProvider) {
+	f.onlyOnce.Do(func() {
+		f.metricsProvider = mp
+	})
+}
+
+func (f *leaderMetricsFactory) newLeaderMetrics() leaderMetricsAdapter {
+	mp := f.metricsProvider
+	if mp == (noopMetricsProvider{}) {
+		return noMetrics{}
+	}
+	return &defaultLeaderMetrics{
+		leader: mp.NewLeaderMetric(),
+	}
+}
+
+// SetProvider sets the metrics provider for all subsequently created work
+// queues. Only the first call has an effect.
+func SetProvider(metricsProvider MetricsProvider) {
+	globalMetricsFactory.setProvider(metricsProvider)
+}