From f1926573804ed2c172c91d1022203d0699210138 Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Tue, 4 Dec 2018 18:20:11 -0800 Subject: [PATCH] Add gauge metric for master of leader election. Fixes #71730 0 indicates standby, 1 indicates master, label indicates which lease. Tweaked name and documentation Factored in Mike Danese feedback. Removed dependency on prometheus from client-go using adapter. Centralized adapter import. Fixed godeps Fixed boilerplate. Put in fixes for caesarxuchao --- cmd/cloud-controller-manager/BUILD | 2 +- .../controller-manager.go | 4 +- cmd/kube-apiserver/BUILD | 2 +- cmd/kube-apiserver/apiserver.go | 4 +- cmd/kube-controller-manager/BUILD | 4 +- .../controller-manager.go | 6 +- cmd/kube-scheduler/BUILD | 2 +- cmd/kube-scheduler/scheduler.go | 4 +- pkg/controller/garbagecollector/BUILD | 2 - .../garbagecollector/garbagecollector.go | 3 - pkg/util/BUILD | 1 + pkg/util/prometheusclientgo/BUILD | 31 +++++ pkg/util/prometheusclientgo/adapters.go | 25 ++++ .../prometheusclientgo/leaderelection/BUILD | 26 +++++ .../leaderelection/adapter.go | 55 +++++++++ plugin/pkg/admission/resourcequota/BUILD | 2 - .../pkg/admission/resourcequota/controller.go | 2 - .../client-go/tools/leaderelection/BUILD | 1 + .../tools/leaderelection/leaderelection.go | 15 ++- .../client-go/tools/leaderelection/metrics.go | 109 ++++++++++++++++++ 20 files changed, 271 insertions(+), 29 deletions(-) create mode 100644 pkg/util/prometheusclientgo/BUILD create mode 100644 pkg/util/prometheusclientgo/adapters.go create mode 100644 pkg/util/prometheusclientgo/leaderelection/BUILD create mode 100644 pkg/util/prometheusclientgo/leaderelection/adapter.go create mode 100644 staging/src/k8s.io/client-go/tools/leaderelection/metrics.go diff --git a/cmd/cloud-controller-manager/BUILD b/cmd/cloud-controller-manager/BUILD index e76c21f221..76dd4b9b99 100644 --- a/cmd/cloud-controller-manager/BUILD +++ b/cmd/cloud-controller-manager/BUILD @@ -20,8 +20,8 @@ go_library( importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager", deps = [ "//cmd/cloud-controller-manager/app:go_default_library", - "//pkg/client/metrics/prometheus:go_default_library", "//pkg/cloudprovider/providers:go_default_library", + "//pkg/util/prometheusclientgo:go_default_library", "//pkg/version/prometheus:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", ], diff --git a/cmd/cloud-controller-manager/controller-manager.go b/cmd/cloud-controller-manager/controller-manager.go index 280069e4d8..61418fae34 100644 --- a/cmd/cloud-controller-manager/controller-manager.go +++ b/cmd/cloud-controller-manager/controller-manager.go @@ -27,11 +27,11 @@ import ( "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/cloud-controller-manager/app" - _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration // NOTE: Importing all in-tree cloud-providers is not required when // implementing an out-of-tree cloud-provider. _ "k8s.io/kubernetes/pkg/cloudprovider/providers" - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration + _ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins + _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration ) func main() { diff --git a/cmd/kube-apiserver/BUILD b/cmd/kube-apiserver/BUILD index 5d7baa2632..d36ddcf4b3 100644 --- a/cmd/kube-apiserver/BUILD +++ b/cmd/kube-apiserver/BUILD @@ -20,7 +20,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kube-apiserver", deps = [ "//cmd/kube-apiserver/app:go_default_library", - "//pkg/client/metrics/prometheus:go_default_library", + "//pkg/util/prometheusclientgo:go_default_library", "//pkg/version/prometheus:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", diff --git a/cmd/kube-apiserver/apiserver.go b/cmd/kube-apiserver/apiserver.go index a73077593e..0591398b90 100644 --- a/cmd/kube-apiserver/apiserver.go +++ b/cmd/kube-apiserver/apiserver.go @@ -27,8 +27,8 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-apiserver/app" - _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration + _ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins + _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration ) func main() { diff --git a/cmd/kube-controller-manager/BUILD b/cmd/kube-controller-manager/BUILD index e21b302681..450c52d237 100644 --- a/cmd/kube-controller-manager/BUILD +++ b/cmd/kube-controller-manager/BUILD @@ -20,9 +20,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kube-controller-manager", deps = [ "//cmd/kube-controller-manager/app:go_default_library", - "//pkg/client/metrics/prometheus:go_default_library", - "//pkg/util/reflector/prometheus:go_default_library", - "//pkg/util/workqueue/prometheus:go_default_library", + "//pkg/util/prometheusclientgo:go_default_library", "//pkg/version/prometheus:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", ], diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go index 7ae1192f97..9923f4b879 100644 --- a/cmd/kube-controller-manager/controller-manager.go +++ b/cmd/kube-controller-manager/controller-manager.go @@ -28,10 +28,8 @@ import ( "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-controller-manager/app" - _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration - _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration - _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration + _ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugin + _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration ) func main() { diff --git a/cmd/kube-scheduler/BUILD b/cmd/kube-scheduler/BUILD index e0c6965fc5..1f541fecb8 100644 --- a/cmd/kube-scheduler/BUILD +++ b/cmd/kube-scheduler/BUILD @@ -20,7 +20,7 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kube-scheduler", deps = [ "//cmd/kube-scheduler/app:go_default_library", - "//pkg/client/metrics/prometheus:go_default_library", + "//pkg/util/prometheusclientgo:go_default_library", "//pkg/version/prometheus:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library", diff --git a/cmd/kube-scheduler/scheduler.go b/cmd/kube-scheduler/scheduler.go index 5d44ed612c..f7b955a4a3 100644 --- a/cmd/kube-scheduler/scheduler.go +++ b/cmd/kube-scheduler/scheduler.go @@ -27,8 +27,8 @@ import ( utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-scheduler/app" - _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration - _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration + _ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins + _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration ) func main() { diff --git a/pkg/controller/garbagecollector/BUILD b/pkg/controller/garbagecollector/BUILD index c1c7f93abb..0b48d84728 100644 --- a/pkg/controller/garbagecollector/BUILD +++ b/pkg/controller/garbagecollector/BUILD @@ -22,8 +22,6 @@ go_library( deps = [ "//pkg/controller:go_default_library", "//pkg/controller/garbagecollector/metaonly:go_default_library", - "//pkg/util/reflector/prometheus:go_default_library", - "//pkg/util/workqueue/prometheus:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/controller/garbagecollector/garbagecollector.go b/pkg/controller/garbagecollector/garbagecollector.go index e090a84fb5..710aa4d6f3 100644 --- a/pkg/controller/garbagecollector/garbagecollector.go +++ b/pkg/controller/garbagecollector/garbagecollector.go @@ -39,9 +39,6 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/controller" - _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration - // install the prometheus plugin - _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // import known versions _ "k8s.io/client-go/kubernetes" ) diff --git a/pkg/util/BUILD b/pkg/util/BUILD index 497e36aadb..e46513b965 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -47,6 +47,7 @@ filegroup( "//pkg/util/parsers:all-srcs", "//pkg/util/pod:all-srcs", "//pkg/util/procfs:all-srcs", + "//pkg/util/prometheusclientgo:all-srcs", "//pkg/util/reflector/prometheus:all-srcs", "//pkg/util/removeall:all-srcs", "//pkg/util/resizefs:all-srcs", diff --git a/pkg/util/prometheusclientgo/BUILD b/pkg/util/prometheusclientgo/BUILD new file mode 100644 index 0000000000..82b05d6895 --- /dev/null +++ b/pkg/util/prometheusclientgo/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["adapters.go"], + importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo", + visibility = ["//visibility:public"], + deps = [ + "//pkg/client/metrics/prometheus:go_default_library", + "//pkg/util/prometheusclientgo/leaderelection:go_default_library", + "//pkg/util/reflector/prometheus:go_default_library", + "//pkg/util/workqueue/prometheus:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/util/prometheusclientgo/leaderelection:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/util/prometheusclientgo/adapters.go b/pkg/util/prometheusclientgo/adapters.go new file mode 100644 index 0000000000..32920cfe42 --- /dev/null +++ b/pkg/util/prometheusclientgo/adapters.go @@ -0,0 +1,25 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheusclientgo + +// Provided metrics needing adapting +import ( + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration + _ "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection" // for leader election metric registration + _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration + _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration +) diff --git a/pkg/util/prometheusclientgo/leaderelection/BUILD b/pkg/util/prometheusclientgo/leaderelection/BUILD new file mode 100644 index 0000000000..b43cf980de --- /dev/null +++ b/pkg/util/prometheusclientgo/leaderelection/BUILD @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["adapter.go"], + importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/util/prometheusclientgo/leaderelection/adapter.go b/pkg/util/prometheusclientgo/leaderelection/adapter.go new file mode 100644 index 0000000000..58753c2900 --- /dev/null +++ b/pkg/util/prometheusclientgo/leaderelection/adapter.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/tools/leaderelection" +) + +// Package prometheus sets the workqueue DefaultMetricsFactory to produce +// prometheus metrics. To use this package, you just have to import it. + +func init() { + leaderelection.SetProvider(prometheusMetricsProvider{}) +} + +type prometheusMetricsProvider struct{} + +func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric { + leaderGauge := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "leader_election_master_status", + Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, + []string{"name"}, + ) + prometheus.Register(leaderGauge) + return &switchAdapter{gauge: leaderGauge} +} + +type switchAdapter struct { + gauge *prometheus.GaugeVec +} + +func (s *switchAdapter) On(name string) { + s.gauge.WithLabelValues(name).Set(1.0) +} + +func (s *switchAdapter) Off(name string) { + s.gauge.WithLabelValues(name).Set(0.0) +} diff --git a/plugin/pkg/admission/resourcequota/BUILD b/plugin/pkg/admission/resourcequota/BUILD index 99c73f482f..3e26346847 100644 --- a/plugin/pkg/admission/resourcequota/BUILD +++ b/plugin/pkg/admission/resourcequota/BUILD @@ -20,8 +20,6 @@ go_library( "//pkg/kubeapiserver/admission:go_default_library", "//pkg/quota/v1:go_default_library", "//pkg/quota/v1/generic:go_default_library", - "//pkg/util/reflector/prometheus:go_default_library", - "//pkg/util/workqueue/prometheus:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota/install:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota/v1beta1:go_default_library", diff --git a/plugin/pkg/admission/resourcequota/controller.go b/plugin/pkg/admission/resourcequota/controller.go index 13f46d1d5c..1c109d2713 100644 --- a/plugin/pkg/admission/resourcequota/controller.go +++ b/plugin/pkg/admission/resourcequota/controller.go @@ -37,8 +37,6 @@ import ( "k8s.io/client-go/util/workqueue" quota "k8s.io/kubernetes/pkg/quota/v1" "k8s.io/kubernetes/pkg/quota/v1/generic" - _ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration - _ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota" ) diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/BUILD b/staging/src/k8s.io/client-go/tools/leaderelection/BUILD index bed0f89fee..d91d57d055 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/BUILD +++ b/staging/src/k8s.io/client-go/tools/leaderelection/BUILD @@ -11,6 +11,7 @@ go_library( srcs = [ "healthzadaptor.go", "leaderelection.go", + "metrics.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection", importpath = "k8s.io/client-go/tools/leaderelection", diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index 2096a5996b..19c57f316e 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -89,10 +89,13 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) { if lec.Lock == nil { return nil, fmt.Errorf("Lock must not be nil.") } - return &LeaderElector{ - config: lec, - clock: clock.RealClock{}, - }, nil + le := LeaderElector{ + config: lec, + clock: clock.RealClock{}, + metrics: globalMetricsFactory.newLeaderMetrics(), + } + le.metrics.leaderOff(le.config.Name) + return &le, nil } type LeaderElectionConfig struct { @@ -152,6 +155,8 @@ type LeaderElector struct { // clock is wrapper around time to allow for less flaky testing clock clock.Clock + metrics leaderMetricsAdapter + // name is the name of the resource lock for debugging name string } @@ -211,6 +216,7 @@ func (le *LeaderElector) acquire(ctx context.Context) bool { return } le.config.Lock.RecordEvent("became leader") + le.metrics.leaderOn(le.config.Name) klog.Infof("successfully acquired lease %v", desc) cancel() }, le.config.RetryPeriod, JitterFactor, true, ctx.Done()) @@ -246,6 +252,7 @@ func (le *LeaderElector) renew(ctx context.Context) { return } le.config.Lock.RecordEvent("stopped leading") + le.metrics.leaderOff(le.config.Name) klog.Infof("failed to renew lease %v: %v", desc, err) cancel() }, le.config.RetryPeriod, ctx.Done()) diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go b/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go new file mode 100644 index 0000000000..65917bf88e --- /dev/null +++ b/staging/src/k8s.io/client-go/tools/leaderelection/metrics.go @@ -0,0 +1,109 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package leaderelection + +import ( + "sync" +) + +// This file provides abstractions for setting the provider (e.g., prometheus) +// of metrics. + +type leaderMetricsAdapter interface { + leaderOn(name string) + leaderOff(name string) +} + +// GaugeMetric represents a single numerical value that can arbitrarily go up +// and down. +type SwitchMetric interface { + On(name string) + Off(name string) +} + +type noopMetric struct{} + +func (noopMetric) On(name string) {} +func (noopMetric) Off(name string) {} + +// defaultLeaderMetrics expects the caller to lock before setting any metrics. +type defaultLeaderMetrics struct { + // leader's value indicates if the current process is the owner of name lease + leader SwitchMetric +} + +func (m *defaultLeaderMetrics) leaderOn(name string) { + if m == nil { + return + } + m.leader.On(name) +} + +func (m *defaultLeaderMetrics) leaderOff(name string) { + if m == nil { + return + } + m.leader.Off(name) +} + +type noMetrics struct{} + +func (noMetrics) leaderOn(name string) {} +func (noMetrics) leaderOff(name string) {} + +// MetricsProvider generates various metrics used by the leader election. +type MetricsProvider interface { + NewLeaderMetric() SwitchMetric +} + +type noopMetricsProvider struct{} + +func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric { + return noopMetric{} +} + +var globalMetricsFactory = leaderMetricsFactory{ + metricsProvider: noopMetricsProvider{}, +} + +type leaderMetricsFactory struct { + metricsProvider MetricsProvider + + onlyOnce sync.Once +} + +func (f *leaderMetricsFactory) setProvider(mp MetricsProvider) { + f.onlyOnce.Do(func() { + f.metricsProvider = mp + }) +} + +func (f *leaderMetricsFactory) newLeaderMetrics() leaderMetricsAdapter { + mp := f.metricsProvider + if mp == (noopMetricsProvider{}) { + return noMetrics{} + } + return &defaultLeaderMetrics{ + leader: mp.NewLeaderMetric(), + } +} + +// SetProvider sets the metrics provider for all subsequently created work +// queues. Only the first call has an effect. +func SetProvider(metricsProvider MetricsProvider) { + globalMetricsFactory.setProvider(metricsProvider) +}