github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add gauge metric for master of leader election. 

Fixes #71730
0 indicates standby, 1 indicates master, label indicates which lease.
Tweaked name and documentation
Factored in Mike Danese feedback.
Removed dependency on prometheus from client-go using adapter.
Centralized adapter import.
Fixed godeps
Fixed boilerplate.
Put in fixes for caesarxuchao
pull/564/head
Walter Fender 2018-12-04 18:20:11 -08:00
parent 68451f301b
commit f192657380
20 changed files with 271 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/cloud-controller-manager/BUILD
View File

@ -20,8 +20,8 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
deps = [
"//cmd/cloud-controller-manager/app:go_default_library",
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/cloudprovider/providers:go_default_library",
"//pkg/util/prometheusclientgo:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
],

4
cmd/cloud-controller-manager/controller-manager.go
View File

@ -27,11 +27,11 @@ import (
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
// NOTE: Importing all in-tree cloud-providers is not required when
// implementing an out-of-tree cloud-provider.
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
func main() {

2
cmd/kube-apiserver/BUILD
View File

@ -20,7 +20,7 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
deps = [
"//cmd/kube-apiserver/app:go_default_library",
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/util/prometheusclientgo:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

4
cmd/kube-apiserver/apiserver.go
View File

@ -27,8 +27,8 @@ import (
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
func main() {

4
cmd/kube-controller-manager/BUILD
View File

@ -20,9 +20,7 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kube-controller-manager",
deps = [
"//cmd/kube-controller-manager/app:go_default_library",
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/util/reflector/prometheus:go_default_library",
"//pkg/util/workqueue/prometheus:go_default_library",
"//pkg/util/prometheusclientgo:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",
],

6
cmd/kube-controller-manager/controller-manager.go
View File

@ -28,10 +28,8 @@ import (
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/kube-controller-manager/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugin
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
func main() {

2
cmd/kube-scheduler/BUILD
View File

@ -20,7 +20,7 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
deps = [
"//cmd/kube-scheduler/app:go_default_library",
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/util/prometheusclientgo:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/logs:go_default_library",

4
cmd/kube-scheduler/scheduler.go
View File

@ -27,8 +27,8 @@ import (
utilflag "k8s.io/apiserver/pkg/util/flag"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/kube-scheduler/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
func main() {

2
pkg/controller/garbagecollector/BUILD
View File

@ -22,8 +22,6 @@ go_library(
deps = [
"//pkg/controller:go_default_library",
"//pkg/controller/garbagecollector/metaonly:go_default_library",
"//pkg/util/reflector/prometheus:go_default_library",
"//pkg/util/workqueue/prometheus:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

3
pkg/controller/garbagecollector/garbagecollector.go
View File

@ -39,9 +39,6 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/util/workqueue"
"k8s.io/kubernetes/pkg/controller"
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
// install the prometheus plugin
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus"
// import known versions
_ "k8s.io/client-go/kubernetes"
)

1
pkg/util/BUILD
View File

@ -47,6 +47,7 @@ filegroup(
"//pkg/util/parsers:all-srcs",
"//pkg/util/pod:all-srcs",
"//pkg/util/procfs:all-srcs",
"//pkg/util/prometheusclientgo:all-srcs",
"//pkg/util/reflector/prometheus:all-srcs",
"//pkg/util/removeall:all-srcs",
"//pkg/util/resizefs:all-srcs",

31
pkg/util/prometheusclientgo/BUILD Normal file
View File

@ -0,0 +1,31 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["adapters.go"],
importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo",
visibility = ["//visibility:public"],
deps = [
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/util/prometheusclientgo/leaderelection:go_default_library",
"//pkg/util/reflector/prometheus:go_default_library",
"//pkg/util/workqueue/prometheus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//pkg/util/prometheusclientgo/leaderelection:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

25
pkg/util/prometheusclientgo/adapters.go Normal file
View File

@ -0,0 +1,25 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prometheusclientgo
// Provided metrics needing adapting
import (
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection" // for leader election metric registration
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
)

26
pkg/util/prometheusclientgo/leaderelection/BUILD Normal file
View File

@ -0,0 +1,26 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["adapter.go"],
importpath = "k8s.io/kubernetes/pkg/util/prometheusclientgo/leaderelection",
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

55
pkg/util/prometheusclientgo/leaderelection/adapter.go Normal file
View File

@ -0,0 +1,55 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prometheus
import (
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/tools/leaderelection"
)
// Package prometheus sets the workqueue DefaultMetricsFactory to produce
// prometheus metrics. To use this package, you just have to import it.
func init() {
leaderelection.SetProvider(prometheusMetricsProvider{})
}
type prometheusMetricsProvider struct{}
func (prometheusMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric {
leaderGauge := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "leader_election_master_status",
Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
},
[]string{"name"},
)
prometheus.Register(leaderGauge)
return &switchAdapter{gauge: leaderGauge}
}
type switchAdapter struct {
gauge *prometheus.GaugeVec
}
func (s *switchAdapter) On(name string) {
s.gauge.WithLabelValues(name).Set(1.0)
}
func (s *switchAdapter) Off(name string) {
s.gauge.WithLabelValues(name).Set(0.0)
}

2
plugin/pkg/admission/resourcequota/BUILD
View File

@ -20,8 +20,6 @@ go_library(
"//pkg/kubeapiserver/admission:go_default_library",
"//pkg/quota/v1:go_default_library",
"//pkg/quota/v1/generic:go_default_library",
"//pkg/util/reflector/prometheus:go_default_library",
"//pkg/util/workqueue/prometheus:go_default_library",
"//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library",
"//plugin/pkg/admission/resourcequota/apis/resourcequota/install:go_default_library",
"//plugin/pkg/admission/resourcequota/apis/resourcequota/v1beta1:go_default_library",

2
plugin/pkg/admission/resourcequota/controller.go
View File

@ -37,8 +37,6 @@ import (
"k8s.io/client-go/util/workqueue"
quota "k8s.io/kubernetes/pkg/quota/v1"
"k8s.io/kubernetes/pkg/quota/v1/generic"
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota"
)

1
staging/src/k8s.io/client-go/tools/leaderelection/BUILD
View File

@ -11,6 +11,7 @@ go_library(
srcs = [
"healthzadaptor.go",
"leaderelection.go",
"metrics.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/tools/leaderelection",
importpath = "k8s.io/client-go/tools/leaderelection",

15
staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go
View File

@ -89,10 +89,13 @@ func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
if lec.Lock == nil {
return nil, fmt.Errorf("Lock must not be nil.")
}
return &LeaderElector{
config: lec,
clock: clock.RealClock{},
}, nil
le := LeaderElector{
config: lec,
clock: clock.RealClock{},
metrics: globalMetricsFactory.newLeaderMetrics(),
}
le.metrics.leaderOff(le.config.Name)
return &le, nil
}
type LeaderElectionConfig struct {
@ -152,6 +155,8 @@ type LeaderElector struct {
// clock is wrapper around time to allow for less flaky testing
clock clock.Clock
metrics leaderMetricsAdapter
// name is the name of the resource lock for debugging
name string
}
@ -211,6 +216,7 @@ func (le *LeaderElector) acquire(ctx context.Context) bool {
return
}
le.config.Lock.RecordEvent("became leader")
le.metrics.leaderOn(le.config.Name)
klog.Infof("successfully acquired lease %v", desc)
cancel()
}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
@ -246,6 +252,7 @@ func (le *LeaderElector) renew(ctx context.Context) {
return
}
le.config.Lock.RecordEvent("stopped leading")
le.metrics.leaderOff(le.config.Name)
klog.Infof("failed to renew lease %v: %v", desc, err)
cancel()
}, le.config.RetryPeriod, ctx.Done())

109
staging/src/k8s.io/client-go/tools/leaderelection/metrics.go Normal file
View File

@ -0,0 +1,109 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package leaderelection
import (
"sync"
)
// This file provides abstractions for setting the provider (e.g., prometheus)
// of metrics.
type leaderMetricsAdapter interface {
leaderOn(name string)
leaderOff(name string)
}
// GaugeMetric represents a single numerical value that can arbitrarily go up
// and down.
type SwitchMetric interface {
On(name string)
Off(name string)
}
type noopMetric struct{}
func (noopMetric) On(name string) {}
func (noopMetric) Off(name string) {}
// defaultLeaderMetrics expects the caller to lock before setting any metrics.
type defaultLeaderMetrics struct {
// leader's value indicates if the current process is the owner of name lease
leader SwitchMetric
}
func (m *defaultLeaderMetrics) leaderOn(name string) {
if m == nil {
return
}
m.leader.On(name)
}
func (m *defaultLeaderMetrics) leaderOff(name string) {
if m == nil {
return
}
m.leader.Off(name)
}
type noMetrics struct{}
func (noMetrics) leaderOn(name string) {}
func (noMetrics) leaderOff(name string) {}
// MetricsProvider generates various metrics used by the leader election.
type MetricsProvider interface {
NewLeaderMetric() SwitchMetric
}
type noopMetricsProvider struct{}
func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric {
return noopMetric{}
}
var globalMetricsFactory = leaderMetricsFactory{
metricsProvider: noopMetricsProvider{},
}
type leaderMetricsFactory struct {
metricsProvider MetricsProvider
onlyOnce sync.Once
}
func (f *leaderMetricsFactory) setProvider(mp MetricsProvider) {
f.onlyOnce.Do(func() {
f.metricsProvider = mp
})
}
func (f *leaderMetricsFactory) newLeaderMetrics() leaderMetricsAdapter {
mp := f.metricsProvider
if mp == (noopMetricsProvider{}) {
return noMetrics{}
}
return &defaultLeaderMetrics{
leader: mp.NewLeaderMetric(),
}
}
// SetProvider sets the metrics provider for all subsequently created work
// queues. Only the first call has an effect.
func SetProvider(metricsProvider MetricsProvider) {
globalMetricsFactory.setProvider(metricsProvider)
}